From d5b1fdf49af8eb23210e3b15a20fe1f9b660eea8 Mon Sep 17 00:00:00 2001 From: Robert Jaszczurek <92210485+rbrtj@users.noreply.github.com> Date: Tue, 17 Sep 2024 09:53:32 +0200 Subject: [PATCH 01/58] [ML] Anomaly Explorer: Display markers for scheduled events in distribution type anomaly charts (#192377) ## Summary Fix for [#129304](https://github.com/elastic/kibana/issues/129304) Previously, for distribution type charts, we displayed calendar event markers only for anomalous data points. The changes improve the display of event markers for such chart types, including showing calendar event markers even when there is no underlying data point. | Scenario | Before | After | | :---: | :---: | :---: | | Rare chart | ![image](https://github.com/user-attachments/assets/c3e186c0-0ec8-434f-a845-3f9e703431dd) | ![image](https://github.com/user-attachments/assets/3dd51cd1-6972-4343-bbc8-8e5f38d7c6bd) | | Population chart | ![Zrzut ekranu 2024-09-9 o 16 16 01](https://github.com/user-attachments/assets/df22dc40-3c8b-46fe-9a5a-02a41278245c) | ![image](https://github.com/user-attachments/assets/c198e75e-14c8-4194-9d71-2358d25f21d5) | | Single metric chart (no difference) | ![image](https://github.com/user-attachments/assets/d0546ba0-46b1-4d2e-9976-fe49bcd4d2da) | ![image](https://github.com/user-attachments/assets/c11ec696-b1f4-4ddf-9542-037b8dd2d31f) | --- x-pack/plugins/ml/common/constants/charts.ts | 2 ++ .../explorer_chart_distribution.js | 32 ++++++++++++++++--- .../models/results_service/anomaly_charts.ts | 16 ++++++++-- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/ml/common/constants/charts.ts b/x-pack/plugins/ml/common/constants/charts.ts index 046504a44177a..b2e9a8493b56f 100644 --- a/x-pack/plugins/ml/common/constants/charts.ts +++ b/x-pack/plugins/ml/common/constants/charts.ts @@ -13,3 +13,5 @@ export const CHART_TYPE = { } as const; export type ChartType = (typeof CHART_TYPE)[keyof typeof CHART_TYPE]; + +export const SCHEDULE_EVENT_MARKER_ENTITY = 'schedule_event_marker_entity'; diff --git a/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_chart_distribution.js b/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_chart_distribution.js index 58052f5f35a65..79e8bd449876e 100644 --- a/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_chart_distribution.js +++ b/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_chart_distribution.js @@ -40,6 +40,7 @@ import { CHART_TYPE } from '../explorer_constants'; import { CHART_HEIGHT, TRANSPARENT_BACKGROUND } from './constants'; import { filter } from 'rxjs'; import { drawCursor } from './utils/draw_anomaly_explorer_charts_cursor'; +import { SCHEDULE_EVENT_MARKER_ENTITY } from '../../../../common/constants/charts'; const CONTENT_WRAPPER_HEIGHT = 215; const SCHEDULED_EVENT_MARKER_HEIGHT = 5; @@ -158,12 +159,29 @@ export class ExplorerChartDistribution extends React.Component { .key((d) => d.entity) .entries(chartData) .sort((a, b) => { + // To display calendar event markers we populate the chart with fake data points. + // If a category has fake data points, it should be sorted to the end. + const aHasFakeData = a.values.some((d) => d.entity === SCHEDULE_EVENT_MARKER_ENTITY); + const bHasFakeData = b.values.some((d) => d.entity === SCHEDULE_EVENT_MARKER_ENTITY); + + if (aHasFakeData && !bHasFakeData) { + return 1; + } + + if (bHasFakeData && !aHasFakeData) { + return -1; + } + return b.values.length - a.values.length; }) .filter((d, i) => { // only filter for rare charts if (chartType === CHART_TYPE.EVENT_DISTRIBUTION) { - return i < categoryLimit || d.key === highlight; + return ( + i < categoryLimit || + d.key === highlight || + d.values.some((d) => d.entity === SCHEDULE_EVENT_MARKER_ENTITY) + ); } return true; }) @@ -373,7 +391,8 @@ export class ExplorerChartDistribution extends React.Component { .orient('left') .innerTickSize(0) .outerTickSize(0) - .tickPadding(10); + .tickPadding(10) + .tickFormat((d) => (d === SCHEDULE_EVENT_MARKER_ENTITY ? null : d)); if (fieldFormat !== undefined) { yAxis.tickFormat((d) => fieldFormat.convert(d, 'text')); @@ -518,7 +537,8 @@ export class ExplorerChartDistribution extends React.Component { const tooltipData = [{ label: formattedDate }]; const seriesKey = config.detectorLabel; - if (marker.entity !== undefined) { + // Hide entity for scheduled events with mocked value. + if (marker.entity !== undefined && marker.entity !== SCHEDULE_EVENT_MARKER_ENTITY) { tooltipData.push({ label: i18n.translate('xpack.ml.explorer.distributionChart.entityLabel', { defaultMessage: 'entity', @@ -590,7 +610,11 @@ export class ExplorerChartDistribution extends React.Component { }); } } - } else if (chartType !== CHART_TYPE.EVENT_DISTRIBUTION) { + } else if ( + chartType !== CHART_TYPE.EVENT_DISTRIBUTION && + // Hide value for scheduled events with mocked value. + marker.entity !== SCHEDULE_EVENT_MARKER_ENTITY + ) { tooltipData.push({ label: i18n.translate( 'xpack.ml.explorer.distributionChart.valueWithoutAnomalyScoreLabel', diff --git a/x-pack/plugins/ml/server/models/results_service/anomaly_charts.ts b/x-pack/plugins/ml/server/models/results_service/anomaly_charts.ts index 8d14417a1d383..7e551b0624261 100644 --- a/x-pack/plugins/ml/server/models/results_service/anomaly_charts.ts +++ b/x-pack/plugins/ml/server/models/results_service/anomaly_charts.ts @@ -53,7 +53,7 @@ import { parseInterval } from '../../../common/util/parse_interval'; import { getDatafeedAggregations } from '../../../common/util/datafeed_utils'; import { findAggField } from '../../../common/util/validation_utils'; import type { ChartType } from '../../../common/constants/charts'; -import { CHART_TYPE } from '../../../common/constants/charts'; +import { CHART_TYPE, SCHEDULE_EVENT_MARKER_ENTITY } from '../../../common/constants/charts'; import { getChartType } from '../../../common/util/chart_utils'; import type { MlJob } from '../..'; @@ -1124,9 +1124,19 @@ export function anomalyChartsDataProvider(mlClient: MlClient, client: IScopedClu each(scheduledEvents, (events, time) => { const chartPoint = findChartPointForTime(chartDataForPointSearch, Number(time)); if (chartPoint !== undefined) { - // Note if the scheduled event coincides with an absence of the underlying metric data, - // we don't worry about plotting the event. chartPoint.scheduledEvents = events; + // We do not want to create additional points for single metric charts + // as it could break the chart. + } else if (chartType !== CHART_TYPE.SINGLE_METRIC) { + // If there's no underlying metric data point for the scheduled event, + // create a new chart point with a value of 0. + const eventChartPoint: ChartPoint = { + date: Number(time), + value: 0, + entity: SCHEDULE_EVENT_MARKER_ENTITY, + scheduledEvents: events, + }; + chartData.push(eventChartPoint); } }); } From 1ae7548c655c47f045923b8671ff8c61b04ca154 Mon Sep 17 00:00:00 2001 From: Tomasz Ciecierski Date: Tue, 17 Sep 2024 10:00:38 +0200 Subject: [PATCH 02/58] [EDR Workflows] Crowdstrike - Add more routes and responses to mocked server (#192881) --- .../emulator_plugins/crowdstrike/mocks.ts | 148 ++++--- .../routes/batch_init_rtr_route.ts | 99 +++++ .../routes/batch_refresh_rtr_session_route.ts | 56 +++ .../routes/batch_rtr_command_route.ts | 169 +++++++ .../routes/get_agent_details_route.ts | 23 + .../routes/get_rtr_command_details_route.ts | 61 +++ .../routes/get_scripts_details_route.ts | 80 ++++ .../routes/get_scripts_ids_route.ts | 36 ++ .../crowdstrike/routes/get_token_route.ts | 13 +- .../crowdstrike/routes/host_actions_route.ts | 26 +- .../crowdstrike/routes/index.ts | 20 + .../crowdstrike/routes/init_rtr_route.ts | 413 ++++++++++++++++++ .../routes/refresh_rtr_session_route.ts | 392 +++++++++++++++++ .../crowdstrike/routes/rtr_admin_route.ts | 264 +++++++++++ .../crowdstrike/routes/rtr_command_route.ts | 137 ++++++ .../crowdstrike/routes/utils.ts | 7 + 16 files changed, 1875 insertions(+), 69 deletions(-) create mode 100644 x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/batch_init_rtr_route.ts create mode 100644 x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/batch_refresh_rtr_session_route.ts create mode 100644 x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/batch_rtr_command_route.ts create mode 100644 x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/get_rtr_command_details_route.ts create mode 100644 x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/get_scripts_details_route.ts create mode 100644 x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/get_scripts_ids_route.ts create mode 100644 x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/init_rtr_route.ts create mode 100644 x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/refresh_rtr_session_route.ts create mode 100644 x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/rtr_admin_route.ts create mode 100644 x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/rtr_command_route.ts diff --git a/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/mocks.ts b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/mocks.ts index e915a16c250b0..d79d683877524 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/mocks.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/mocks.ts @@ -10,6 +10,7 @@ import type { CrowdstrikeGetAgentsResponse, } from '@kbn/stack-connectors-plugin/common/crowdstrike/types'; import { merge } from 'lodash'; +import { TEST_AGENT_ID, TEST_CID_ID } from './routes/utils'; export const createCrowdstrikeGetAgentsApiResponseMock = ( data: CrowdstrikeBaseApiResponse['resources'][number] @@ -18,55 +19,66 @@ export const createCrowdstrikeGetAgentsApiResponseMock = ( meta: { query_time: 0.001831479, powered_by: 'device-api', - trace_id: '4567898765-432423432432-42342342', + trace_id: 'xxx', }, errors: null, resources: data, }; }; +export const createCrowdstrikeErrorResponseMock = (error: object) => { + return { + meta: { + query_time: 0.001831479, + powered_by: 'device-api', + trace_id: 'xxx', + }, + errors: [error], + }; +}; + export const createCrowdstrikeAgentDetailsMock = ( overrides: Partial = {} ): CrowdstrikeGetAgentsResponse['resources'][number] => { return merge( { - device_id: '5f4ed7ec2690431f8fa79213268779cb', - cid: '234567890', + device_id: TEST_AGENT_ID, + cid: TEST_CID_ID, agent_load_flags: '0', - agent_local_time: '2024-03-18T22:21:00.173Z', - agent_version: '7.07.16206.0', - bios_manufacturer: 'Amazon EC2', - bios_version: '1.0', - config_id_base: '65994753', - config_id_build: '16206', - config_id_platform: '8', - cpu_signature: '8392466', - cpu_vendor: '1', - external_ip: '18.157.150.216', - mac_address: '03-f4-f4-f4-f4', - instance_id: 'i-456789', - service_provider: 'AWS_EC2_V2', - service_provider_account_id: '23456789', - hostname: 'Crowdstrike-1460', - first_seen: '2024-03-15T13:18:56Z', - last_login_timestamp: '2024-03-15T22:11:47Z', - last_login_user: 'testuser', - last_login_uid: '1002', - last_seen: '2024-03-20T07:19:01Z', - local_ip: '172.31.200.45', - major_version: '5', - minor_version: '14', - os_version: 'RHEL 9.3', + agent_local_time: '2024-09-08T06:07:00.326Z', + agent_version: '7.18.17106.0', + bios_manufacturer: 'EFI Development Kit II / OVMF', + bios_version: '0.0.0', + config_id_base: '65994763', + config_id_build: '17106', + config_id_platform: '128', + cpu_signature: '4294967295', + cpu_vendor: '3', + external_ip: '79.184.246.19', + mac_address: '52-54-00-09-42-a6', + hostname: 'cs-falcon', + filesystem_containment_status: 'normal', + first_login_timestamp: '2024-08-19T08:37:15Z', + first_login_user: 'ubuntu', + first_seen: '2024-08-19T08:37:17Z', + last_login_timestamp: '2024-08-19T08:37:15Z', + last_login_user: 'ubuntu', + last_login_uid: '1000', + last_seen: '2024-09-10T09:32:58Z', + local_ip: '192.168.80.7', + major_version: '6', + minor_version: '8', + os_version: 'Ubuntu 24.04', platform_id: '3', platform_name: 'Linux', policies: [ { policy_type: 'prevention', - policy_id: '234234234234', + policy_id: 'test_prevention_policy_id', applied: true, - settings_hash: 'f0e04444', - assigned_date: '2024-03-15T13:20:02.25821602Z', - applied_date: '2024-03-15T13:20:16.804783955Z', + settings_hash: 'test2984', + assigned_date: '2024-08-19T08:40:24.454802663Z', + applied_date: '2024-08-19T08:46:46.169115065Z', rule_groups: [], }, ], @@ -74,61 +86,67 @@ export const createCrowdstrikeAgentDetailsMock = ( device_policies: { prevention: { policy_type: 'prevention', - policy_id: '234234234234', + policy_id: 'test_prevention_policy_id', applied: true, - settings_hash: 'f0e04444', - assigned_date: '2024-03-15T13:20:02.25821602Z', - applied_date: '2024-03-15T13:20:16.804783955Z', + settings_hash: 'test2984', + assigned_date: '2024-08-19T08:40:24.454802663Z', + applied_date: '2024-08-19T08:46:46.169115065Z', rule_groups: [], }, sensor_update: { policy_type: 'sensor-update', - policy_id: '234234234234', + policy_id: 'test_sensor_update_policy_id', applied: true, - settings_hash: 'tagged|5;', - assigned_date: '2024-03-15T13:20:02.258765734Z', - applied_date: '2024-03-15T13:23:53.773752711Z', + settings_hash: 'test3a5bb', + assigned_date: '2024-08-19T08:40:24.406563043Z', + applied_date: '2024-08-19T08:44:54.277815271Z', uninstall_protection: 'UNKNOWN', }, global_config: { policy_type: 'globalconfig', - policy_id: '234234234234', + policy_id: 'test_global_config_policy_id', applied: true, - settings_hash: 'f0e04444', - assigned_date: '2024-03-18T22:21:01.50638371Z', - applied_date: '2024-03-18T22:21:30.565040189Z', + settings_hash: 'testa5bc', + assigned_date: '2024-09-08T04:54:07.410501178Z', + applied_date: '2024-09-08T04:55:06.81648557Z', }, remote_response: { policy_type: 'remote-response', - policy_id: '234234234234', + policy_id: 'test_remote_response_policy_id', + applied: true, + settings_hash: 'test205c', + assigned_date: '2024-08-19T08:48:00.144480664Z', + applied_date: '2024-08-19T08:55:01.036602542Z', + }, + 'host-retention': { + policy_type: 'host-retention', + policy_id: 'test_host-retention_policy_id', applied: true, - settings_hash: 'f0e04444', - assigned_date: '2024-03-15T13:20:02.258285018Z', - applied_date: '2024-03-15T13:20:17.016591803Z', + settings_hash: 'testfghjk', + assigned_date: '2024-08-19T08:40:24.444810716Z', + applied_date: '2024-08-19T08:44:54.577562462Z', }, }, - groups: [], - group_hash: '45678909876545678', + groups: ['test123', 'test456'], + group_hash: 'test123', product_type_desc: 'Server', - provision_status: 'NotProvisioned', - serial_number: '345678765-35d6-e704-1723-423423432', - status: 'containment_pending', - system_manufacturer: 'Amazon EC2', - system_product_name: 't3a.medium', + provision_status: 'Provisioned', + status: 'normal', + system_manufacturer: 'QEMU', + system_product_name: 'QEMU Virtual Machine', tags: [], - modified_timestamp: '2024-03-20T07:19:45Z', + modified_timestamp: '2024-09-10T09:33:21Z', meta: { - version: '484', - version_string: '9:33384301139', + version: '552', + version_string: '1:1815077394', }, - zone_group: 'eu-central-1a', - kernel_version: '5.14.0-234234el9_3.x86_64', + kernel_version: '6.8.0-41-generic', chassis_type: '1', chassis_type_desc: 'Other', - connection_ip: '172.31.200.45', - default_gateway_ip: '172.31.200.1', - connection_mac_address: '02-e8-f1-0e-b7-c4', - linux_sensor_mode: 'Kernel Mode', + connection_ip: '192.168.80.7', + default_gateway_ip: '192.168.80.1', + connection_mac_address: '52-54-00-09-42-a6', + linux_sensor_mode: 'User Mode', deployment_type: 'Standard', }, overrides @@ -140,8 +158,10 @@ export const createCrowdstrikeGetAgentOnlineStatusDetailsMock: ( ) => CrowdstrikeGetAgentOnlineStatusResponse['resources'][number] = (overrides) => { return merge( { + id: TEST_AGENT_ID, + cid: TEST_CID_ID, + last_seen: '2024-09-10T09:59:56Z', state: 'online', - id: '5f4ed7ec2690431f8fa79213268779cb', }, overrides ); diff --git a/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/batch_init_rtr_route.ts b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/batch_init_rtr_route.ts new file mode 100644 index 0000000000000..940ad4d127d4e --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/batch_init_rtr_route.ts @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { buildCrowdstrikeRoutePath, TEST_AGENT_ID, TEST_BATCH_ID, TEST_SESSION_ID } from './utils'; +import type { ExternalEdrServerEmulatorRouteHandlerMethod } from '../../../external_edr_server_emulator.types'; +import type { EmulatorServerRouteDefinition } from '../../../lib/emulator_server.types'; + +export const batchInitRTRSessionRoute = (): EmulatorServerRouteDefinition => { + return { + path: buildCrowdstrikeRoutePath('/real-time-response/combined/batch-init-session/v1'), + method: 'POST', + handler: batchInitSessionSuccessHandler, + }; +}; +// @ts-expect-error - example of error response +const initSessionWrongHostIdError = async () => { + return { + meta: { + query_time: 0.244284399, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + batch_id: '', + resources: { + [TEST_AGENT_ID]: { + session_id: '', + complete: false, + stdout: '', + stderr: '', + aid: TEST_AGENT_ID, + errors: [ + { + code: 500, + message: `uuid: incorrect UUID length 47 in string ${TEST_AGENT_ID}`, + }, + ], + query_time: 0, + offline_queued: false, + }, + }, + errors: [ + { + code: 404, + message: 'no successful hosts initialized on RTR', + }, + ], + }; +}; +// @ts-expect-error - example of error response +const initSessionMissingIdsError = async () => { + return { + meta: { + query_time: 0.00034664, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + batch_id: '', + resources: {}, + errors: [ + { + code: 400, + message: + 'Invalid number of hosts in request: 0. Must be an integer greater than 0 and less than or equal to 10000', + }, + ], + }; +}; + +const batchInitSessionSuccessHandler: ExternalEdrServerEmulatorRouteHandlerMethod< + {}, + {} +> = async () => { + return { + meta: { + query_time: 1.067267552, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + batch_id: TEST_BATCH_ID, + resources: { + [TEST_AGENT_ID]: { + session_id: TEST_SESSION_ID, + task_id: 'xxx', + complete: true, + stdout: '/', + stderr: '', + base_command: 'pwd', + aid: TEST_AGENT_ID, + errors: [], + query_time: 0, + offline_queued: false, + }, + }, + errors: [], + }; +}; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/batch_refresh_rtr_session_route.ts b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/batch_refresh_rtr_session_route.ts new file mode 100644 index 0000000000000..0b7ae5b6f4e19 --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/batch_refresh_rtr_session_route.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { buildCrowdstrikeRoutePath, TEST_AGENT_ID, TEST_SESSION_ID } from './utils'; +import type { ExternalEdrServerEmulatorRouteHandlerMethod } from '../../../external_edr_server_emulator.types'; +import type { EmulatorServerRouteDefinition } from '../../../lib/emulator_server.types'; + +export const batchRTRRefreshSessionRoute = (): EmulatorServerRouteDefinition => { + return { + path: buildCrowdstrikeRoutePath('/real-time-response/combined/batch-refresh-session/v1'), + method: 'POST', + handler: batchRTRRefreshSessionHandler, + }; +}; + +// @ts-expect-error - example of error response +const batchRTRRefreshSessionInvalidSessionError = async () => { + return { + meta: { + query_time: 0.001031577, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + resources: {}, + errors: [ + { + code: 400, + message: 'no hosts in this batch session', + }, + ], + }; +}; + +const batchRTRRefreshSessionHandler: ExternalEdrServerEmulatorRouteHandlerMethod< + {}, + {} +> = async () => { + return { + meta: { + query_time: 0.068379923, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + resources: { + [TEST_AGENT_ID]: { + aid: TEST_AGENT_ID, + session_id: TEST_SESSION_ID, + errors: [], + }, + }, + errors: [], + }; +}; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/batch_rtr_command_route.ts b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/batch_rtr_command_route.ts new file mode 100644 index 0000000000000..07f5ad927b3d4 --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/batch_rtr_command_route.ts @@ -0,0 +1,169 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { buildCrowdstrikeRoutePath, TEST_AGENT_ID, TEST_SESSION_ID } from './utils'; +import type { ExternalEdrServerEmulatorRouteHandlerMethod } from '../../../external_edr_server_emulator.types'; +import type { EmulatorServerRouteDefinition } from '../../../lib/emulator_server.types'; + +export const batchRTRCommandRoute = (): EmulatorServerRouteDefinition => { + return { + // we use `combined` api - which returns just one complete response, otherwise it would be coming in batches + path: buildCrowdstrikeRoutePath('/real-time-response/combined/batch-command/v1'), + method: 'POST', + handler: batchRTRCommandSuccessHandler, + }; +}; + +// @ts-expect-error - example of missing file error +const batchCommandResponseWithError = async () => { + return { + meta: { + query_time: 0.913513625, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + combined: { + resources: { + [TEST_AGENT_ID]: { + session_id: TEST_SESSION_ID, + task_id: 'xxx', + complete: true, + stdout: '', + stderr: 'cat: test.xt: No such file or directory', + base_command: 'cat', + aid: TEST_AGENT_ID, + errors: [], + query_time: 0.912058582, + offline_queued: false, + }, + }, + }, + errors: [], + }; +}; + +// @ts-expect-error - example of error response +const batchCommandResponseInvalidCommandError = async () => { + return { + meta: { + query_time: 0.101208469, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + combined: { + resources: { + [TEST_AGENT_ID]: { + session_id: '', + complete: false, + stdout: '', + stderr: '', + aid: TEST_AGENT_ID, + errors: [ + { + code: 40007, + message: 'Command not found', + }, + ], + query_time: 0, + offline_queued: false, + }, + }, + }, + errors: [], + }; +}; + +// @ts-expect-error - example of error response +const batchCommandInvalidSessionError = async () => { + return { + meta: { + query_time: 0.02078217, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + combined: { + resources: { + [TEST_AGENT_ID]: { + session_id: '', + complete: false, + stdout: '', + stderr: '', + aid: TEST_AGENT_ID, + errors: [ + { + code: 50007, + message: 'could not get session', + }, + ], + query_time: 0, + offline_queued: false, + }, + }, + }, + errors: [], + }; +}; +// @ts-expect-error - example of error response +const batchCommandCommandIsNotValidError = async () => { + return { + meta: { + query_time: 0.122372386, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + combined: { + resources: { + [TEST_AGENT_ID]: { + session_id: '', + complete: false, + stdout: '', + stderr: '', + aid: TEST_AGENT_ID, + errors: [ + { + code: 40006, + message: 'Command is not valid', + }, + ], + query_time: 0, + offline_queued: false, + }, + }, + }, + errors: [], + }; +}; + +const batchRTRCommandSuccessHandler: ExternalEdrServerEmulatorRouteHandlerMethod< + {}, + {} +> = async () => { + return { + meta: { + query_time: 0.888750872, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + combined: { + resources: { + [TEST_AGENT_ID]: { + session_id: TEST_SESSION_ID, + task_id: 'xxx', + complete: true, + stdout: + 'bin\nbin.usr-is-merged\nboot\ndev\netc\nhome\nlib\nlib.usr-is-merged\nlost+found\nmedia\nmnt\nopt\nproc\nroot\nrun\nsbin\nsbin.usr-is-merged\nsnap\nsrv\nsys\ntmp\nusr\nvar', + stderr: '', + base_command: 'ls', + aid: TEST_AGENT_ID, + errors: [], + query_time: 0.887764377, + offline_queued: false, + }, + }, + }, + errors: [], + }; +}; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/get_agent_details_route.ts b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/get_agent_details_route.ts index 84e5f42783f3a..1734cda7e745e 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/get_agent_details_route.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/get_agent_details_route.ts @@ -21,6 +21,29 @@ export const getAgentDetailsRouteDefinition = (): EmulatorServerRouteDefinition }; }; +// @ts-expect-error - example of missing file error +const getAgentDetailsMissingIdsError = async () => { + return { + errors: [ + { + code: 400, + message: "The 'ids' parameter must be present at least once.", + }, + ], + }; +}; +// @ts-expect-error - example of missing file error +const getAgentDetailsInvalidIdsError = async () => { + return { + errors: [ + { + code: 400, + message: 'invalid device id [asdasd]', + }, + ], + }; +}; + const getAgentDetailsHandler: ExternalEdrServerEmulatorRouteHandlerMethod<{}> = async () => { return createCrowdstrikeGetAgentsApiResponseMock([createCrowdstrikeAgentDetailsMock({})]); }; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/get_rtr_command_details_route.ts b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/get_rtr_command_details_route.ts new file mode 100644 index 0000000000000..3028d2910c7c4 --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/get_rtr_command_details_route.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { buildCrowdstrikeRoutePath, TEST_SESSION_ID } from './utils'; +import type { ExternalEdrServerEmulatorRouteHandlerMethod } from '../../../external_edr_server_emulator.types'; +import type { EmulatorServerRouteDefinition } from '../../../lib/emulator_server.types'; + +export const getRTRCommandDetailsRoute = (): EmulatorServerRouteDefinition => { + return { + // PARAMS /v1?cloud_request_id=test-cloud-request1&sequence_id=0 + path: buildCrowdstrikeRoutePath('/real-time-response/entities/command/v1'), + method: 'GET', + handler: getRTRCommandDetailsSuccessHandler, + }; +}; + +// @ts-expect-error - example of missing file error +const commandDetailsMissingCloudIdError = async () => { + return { + meta: { + query_time: 0.000238205, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + resources: [], + errors: [ + { + code: 400, + message: 'cloud_request_id must be a uuid string', + }, + ], + }; +}; + +const getRTRCommandDetailsSuccessHandler: ExternalEdrServerEmulatorRouteHandlerMethod< + {}, + {} +> = async () => { + return { + meta: { + query_time: 0.307542055, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + resources: [ + { + session_id: TEST_SESSION_ID, + task_id: 'xxx', + complete: true, + stdout: + 'archive\nbackup\nbin\nboot\ndev\necho\netc\nhome\nlib\nlost+found\nmedia\nmnt\nopt\nproc\nroot\nrun\nsbin\nsnap\nsrv\nstuff.exe\nsys\ntest.sh\ntestPush.exe\ntmp\nusr\nvar\n', + stderr: '', + base_command: 'ls', + }, + ], + errors: [], + }; +}; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/get_scripts_details_route.ts b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/get_scripts_details_route.ts new file mode 100644 index 0000000000000..38f57e1f7b0a8 --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/get_scripts_details_route.ts @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { buildCrowdstrikeRoutePath } from './utils'; +import type { ExternalEdrServerEmulatorRouteHandlerMethod } from '../../../external_edr_server_emulator.types'; +import type { EmulatorServerRouteDefinition } from '../../../lib/emulator_server.types'; + +export const getCustomScriptsDetailsRoute = (): EmulatorServerRouteDefinition => { + return { + path: buildCrowdstrikeRoutePath('/real-time-response/entities/scripts/v1'), + method: 'GET', + handler: getCustomScriptsDetailsSuccessHandler, + }; +}; + +// @ts-expect-error - example of missing file error +const getScriptsIdEmptyResponse = async () => { + return { + meta: { + query_time: 0.025758121, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + resources: [], + }; +}; + +const getCustomScriptsDetailsSuccessHandler: ExternalEdrServerEmulatorRouteHandlerMethod< + {}, + {} +> = async () => { + return { + meta: { + query_time: 0.531831172, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + resources: [ + { + id: 'test-id-1', + name: 'AnyDesk Remediation', + file_type: 'script', + platform: ['windows'], + size: 1979, + content: 'echo "AnyDesk Remediation"', + created_by: 'testuser@test.com', + created_by_uuid: '34d10-a0f7e5dc98a8', + created_timestamp: '2023-08-01T05:20:10.695182885Z', + modified_by: 'testuser@test.com', + modified_timestamp: '2023-08-01T05:20:10.695183074Z', + sha256: 'test58f74e15e56815d71b29450f077df2f6070630184b9d', + permission_type: 'public', + run_attempt_count: 67, + run_success_count: 0, + write_access: true, + }, + { + id: 'test-id-2', + name: 'Gather AnyDesk Artifacts', + file_type: 'script', + platform: ['windows'], + size: 1184, + content: 'echo Gather Anydesk Artifacts', + created_by: 'testuser@test.com', + created_by_uuid: '34d0610-a0f7e5dc98a8', + created_timestamp: '2023-08-17T07:08:00.839412392Z', + modified_by: 'testuser@test.com', + modified_timestamp: '2023-08-17T07:08:00.839412727Z', + sha256: 'teste8dfbb7cfb782c11484b47d336a93fdae80cffa77039c5', + permission_type: 'public', + run_attempt_count: 4, + run_success_count: 0, + write_access: true, + }, + ], + }; +}; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/get_scripts_ids_route.ts b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/get_scripts_ids_route.ts new file mode 100644 index 0000000000000..71a6caf257ad0 --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/get_scripts_ids_route.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { buildCrowdstrikeRoutePath } from './utils'; +import type { ExternalEdrServerEmulatorRouteHandlerMethod } from '../../../external_edr_server_emulator.types'; +import type { EmulatorServerRouteDefinition } from '../../../lib/emulator_server.types'; + +export const getCustomScriptsIdsRoute = (): EmulatorServerRouteDefinition => { + return { + path: buildCrowdstrikeRoutePath('/real-time-response/queries/scripts/v1'), + method: 'GET', + handler: getCustomScriptsIdsHandler, + }; +}; + +const getCustomScriptsIdsHandler: ExternalEdrServerEmulatorRouteHandlerMethod< + {}, + {} +> = async () => { + return { + meta: { + query_time: 0.030241162, + pagination: { + offset: 0, + limit: 100, + total: 11, + }, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + resources: ['test-id-1', 'test-id-2'], + }; +}; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/get_token_route.ts b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/get_token_route.ts index 6629c33974b04..8bc50860beb2a 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/get_token_route.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/get_token_route.ts @@ -4,6 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import { createCrowdstrikeErrorResponseMock } from '../mocks'; import { buildCrowdstrikeRoutePath } from './utils'; import type { ExternalEdrServerEmulatorRouteHandlerMethod } from '../../../external_edr_server_emulator.types'; import type { EmulatorServerRouteDefinition } from '../../../lib/emulator_server.types'; @@ -16,14 +17,18 @@ export const getTokenRouteDefinition = (): EmulatorServerRouteDefinition => { }; }; +// @ts-expect-error - example of missing token error +const getTokenError = async () => { + return createCrowdstrikeErrorResponseMock({ + code: 401, + message: 'access denied, invalid bearer token', + }); +}; + const getTokenHandler: ExternalEdrServerEmulatorRouteHandlerMethod<{}> = async () => { return { access_token: 'testtoken', expires_in: 123, token_type: 'bearer', - id_token: 'test', - issued_token_type: 'test', - refresh_token: 'test', - scope: 'test', }; }; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/host_actions_route.ts b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/host_actions_route.ts index 2c648213a3f09..273285d519ded 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/host_actions_route.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/host_actions_route.ts @@ -17,6 +17,30 @@ export const hostActionsRouteDefinition = (): EmulatorServerRouteDefinition => { }; }; +// @ts-expect-error - example of missing action parameter error +const hostActionsMissingActionParameterError = async () => { + return { + errors: [ + { + code: 400, + message: "Provided data does not match expected 'Action Parameter' format", + }, + ], + }; +}; + +// @ts-expect-error - example of missing agent id error +const hostActionsInvalidAgentIdError = async () => { + return { + errors: [ + { + code: 404, + message: 'No matching device found for ID wrongAgentId', + }, + ], + }; +}; + const hostActionsHandler: ExternalEdrServerEmulatorRouteHandlerMethod< {}, CrowdstrikeHostActionsParams @@ -24,7 +48,7 @@ const hostActionsHandler: ExternalEdrServerEmulatorRouteHandlerMethod< return { resources: [ { - id: 'test', + id: 'test_id', path: 'test', }, ], diff --git a/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/index.ts b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/index.ts index 1da4f435aae4c..3be3fe20a2819 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/index.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/index.ts @@ -5,11 +5,21 @@ * 2.0. */ +import { rtrCommandRoute } from './rtr_command_route'; +import { batchRTRRefreshSessionRoute } from './batch_refresh_rtr_session_route'; +import { rtrAdminCommandRoute } from './rtr_admin_route'; +import { refreshRTRSessionRoute } from './refresh_rtr_session_route'; +import { getCustomScriptsDetailsRoute } from './get_scripts_details_route'; +import { initRTRSessionRoute } from './init_rtr_route'; +import { getRTRCommandDetailsRoute } from './get_rtr_command_details_route'; +import { batchRTRCommandRoute } from './batch_rtr_command_route'; +import { batchInitRTRSessionRoute } from './batch_init_rtr_route'; import { getTokenRouteDefinition } from './get_token_route'; import type { EmulatorServerRouteDefinition } from '../../../lib/emulator_server.types'; import { hostActionsRouteDefinition } from './host_actions_route'; import { getAgentDetailsRouteDefinition } from './get_agent_details_route'; import { getAgentOnlineStatusRouteDefinition } from './get_agent_online_status_route'; +import { getCustomScriptsIdsRoute } from './get_scripts_ids_route'; export const getCrowdstrikeRouteDefinitions = (): EmulatorServerRouteDefinition[] => { return [ @@ -17,5 +27,15 @@ export const getCrowdstrikeRouteDefinitions = (): EmulatorServerRouteDefinition[ hostActionsRouteDefinition(), getAgentDetailsRouteDefinition(), getAgentOnlineStatusRouteDefinition(), + batchInitRTRSessionRoute(), + batchRTRCommandRoute(), + getRTRCommandDetailsRoute(), + getCustomScriptsIdsRoute(), + getCustomScriptsDetailsRoute(), + initRTRSessionRoute(), + refreshRTRSessionRoute(), + rtrAdminCommandRoute(), + batchRTRRefreshSessionRoute(), + rtrCommandRoute(), ]; }; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/init_rtr_route.ts b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/init_rtr_route.ts new file mode 100644 index 0000000000000..f16f166551356 --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/init_rtr_route.ts @@ -0,0 +1,413 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { buildCrowdstrikeRoutePath, TEST_AGENT_ID, TEST_SESSION_ID } from './utils'; +import type { ExternalEdrServerEmulatorRouteHandlerMethod } from '../../../external_edr_server_emulator.types'; +import type { EmulatorServerRouteDefinition } from '../../../lib/emulator_server.types'; + +export const initRTRSessionRoute = (): EmulatorServerRouteDefinition => { + return { + path: buildCrowdstrikeRoutePath('/real-time-response/entities/sessions/v1'), + method: 'POST', + handler: initRTRSessionHandler, + }; +}; +// requestBody: +// { +// "device_id": "xxxxxx", +// "queue_offline": false +// } + +// @ts-expect-error - example of invalid agent id error +const initRTRSessionInvalidAgentIdError = async () => { + return { + meta: { + query_time: 0.244284399, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + batch_id: '', + resources: { + [TEST_AGENT_ID]: { + session_id: '', + complete: false, + stdout: '', + stderr: '', + aid: TEST_AGENT_ID, + errors: [ + { + code: 500, + message: 'uuid: incorrect UUID length 47 in string "wrongAgentId"', + }, + ], + query_time: 0, + offline_queued: false, + }, + }, + errors: [ + { + code: 404, + message: 'no successful hosts initialized on RTR', + }, + ], + }; +}; + +// @ts-expect-error - example of missing agent id error +const initRTRSessionMissingAgentIdError = async () => { + return { + meta: { + query_time: 0.00034664, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + batch_id: '', + resources: {}, + errors: [ + { + code: 400, + message: + 'Invalid number of hosts in request: 0. Must be an integer greater than 0 and less than or equal to 10000', + }, + ], + }; +}; + +const initRTRSessionHandler: ExternalEdrServerEmulatorRouteHandlerMethod<{}, {}> = async () => { + return { + meta: { + query_time: 1.776937422, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + resources: [ + { + session_id: TEST_SESSION_ID, + scripts: [ + { + command: 'cat', + description: 'Read a file from disk and display as ASCII', + examples: 'cat foo.txt\r\ncat -n foo.txt\r\ncat -t foo.txt\r\ncat -t -n foo.txt', + internal_only: false, + runnable: true, + sub_commands: [], + args: [ + { + id: 671, + created_at: '2020-09-22T00:54:20Z', + updated_at: '2020-09-22T00:54:20Z', + script_id: 94, + arg_type: 'arg', + data_type: 'string', + requires_value: false, + arg_name: 'Path', + description: 'path to cat', + default_value: '', + required: true, + sequence: 1, + options: null, + encoding: '', + command_level: 'non-destructive', + }, + { + id: 672, + created_at: '2020-09-22T00:54:20Z', + updated_at: '2020-09-22T00:54:20Z', + script_id: 94, + arg_type: 'flag', + data_type: 'string', + requires_value: false, + arg_name: 'n', + description: 'Number the output lines starting from 1', + default_value: '', + required: false, + sequence: 2, + options: null, + encoding: '', + command_level: 'non-destructive', + }, + { + id: 673, + created_at: '2020-09-22T00:54:20Z', + updated_at: '2020-09-22T00:54:20Z', + script_id: 94, + arg_type: 'flag', + data_type: 'string', + requires_value: false, + arg_name: 't', + description: "Display non-printing characters, and display tab characters as `^I'.", + default_value: '', + required: false, + sequence: 3, + options: null, + encoding: '', + command_level: 'non-destructive', + }, + ], + }, + { + command: 'cd', + description: 'Change the current working directory', + examples: '', + internal_only: false, + runnable: true, + sub_commands: [], + args: [ + { + id: 674, + created_at: '2020-09-22T00:54:07Z', + updated_at: '2020-09-22T00:54:07Z', + script_id: 95, + arg_type: 'arg', + data_type: 'string', + requires_value: false, + arg_name: 'Path', + description: 'path', + default_value: '', + required: true, + sequence: 1, + options: null, + encoding: '', + command_level: 'non-destructive', + }, + ], + }, + { + command: 'env', + description: 'Print out the environment', + examples: 'env', + internal_only: false, + runnable: true, + sub_commands: [], + args: [], + }, + { + command: 'filehash', + description: 'Generate the MD5, SHA1, and SHA256 hashes of a file', + examples: 'filehash /tmp/test', + internal_only: false, + runnable: true, + sub_commands: [], + args: [ + { + id: 680, + created_at: '2020-09-22T00:53:36Z', + updated_at: '2020-09-22T00:53:36Z', + script_id: 100, + arg_type: 'arg', + data_type: 'string', + requires_value: false, + arg_name: 'Path', + description: 'File to hash', + default_value: '', + required: true, + sequence: 1, + options: null, + encoding: '', + command_level: 'non-destructive', + }, + ], + }, + { + command: 'ifconfig', + description: 'Show network configuration information', + examples: 'ifconfig', + internal_only: false, + runnable: true, + sub_commands: [], + args: [], + }, + { + command: 'ls', + description: 'Display the contents of the specified path', + examples: + 'ls\r\nls -l\r\nls -L\r\nls -t\r\nls -l -@\r\nls -R\r\nls -l -R\r\nls -l -t -R -L', + internal_only: false, + runnable: true, + sub_commands: [], + args: [ + { + id: 684, + created_at: '2020-09-22T00:53:14Z', + updated_at: '2020-09-22T00:53:14Z', + script_id: 104, + arg_type: 'arg', + data_type: 'string', + requires_value: false, + arg_name: 'Path', + description: 'Path', + default_value: '.', + required: false, + sequence: 1, + options: null, + encoding: '', + command_level: 'non-destructive', + }, + { + id: 685, + created_at: '2020-09-22T00:53:14Z', + updated_at: '2020-09-22T00:53:14Z', + script_id: 104, + arg_type: 'flag', + data_type: 'string', + requires_value: false, + arg_name: 'l', + description: 'List in long format.', + default_value: '', + required: false, + sequence: 2, + options: null, + encoding: '', + command_level: 'non-destructive', + }, + { + id: 686, + created_at: '2020-09-22T00:53:14Z', + updated_at: '2020-09-22T00:53:14Z', + script_id: 104, + arg_type: 'flag', + data_type: 'string', + requires_value: false, + arg_name: 'L', + description: + 'Follow all symbolic links to final target and list the file or directory the link references rather than the link itself.', + default_value: '', + required: false, + sequence: 3, + options: null, + encoding: '', + command_level: 'non-destructive', + }, + { + id: 687, + created_at: '2020-09-22T00:53:14Z', + updated_at: '2020-09-22T00:53:14Z', + script_id: 104, + arg_type: 'flag', + data_type: 'string', + requires_value: false, + arg_name: 'R', + description: 'Recursively list subdirectories encountered.', + default_value: '', + required: false, + sequence: 4, + options: null, + encoding: '', + command_level: 'non-destructive', + }, + { + id: 688, + created_at: '2020-09-22T00:53:14Z', + updated_at: '2020-09-22T00:53:14Z', + script_id: 104, + arg_type: 'flag', + data_type: 'string', + requires_value: false, + arg_name: 't', + description: + 'Sort by time modified (most recently modified first) before sorting the operands by lexicographical order.', + default_value: '', + required: false, + sequence: 5, + options: null, + encoding: '', + command_level: 'non-destructive', + }, + ], + }, + { + command: 'mount', + description: 'List or mount filesystem volumes', + examples: + 'Executable by all RTR roles:\r\nmount\r\nExecutable by privileged RTR users only:\r\nmount -t=nfs "host:/exports/filesystem" "/mnt/filesystem"\r\n Mount the NFS filesystem located at "/exports/filesystem" on "host" to the local destination "/mnt/filesystem"\r\nmount -t=smbfs "//user:password@host/filesystem" "/mnt/mountpoint"\r\n Mount the SMB "/filesystem" on "host" as "user" with "password" to "/mnt/mountpoint"\r\nmount -t=smbfs -o=nobrowse "//user:password@host/filesystem" "/mnt/mountpoint"\r\n Mount the SMB "/filesystem" with option "nobrowse" on "host" as "user" with "password" to "/mnt/mountpoint"', + internal_only: false, + runnable: true, + sub_commands: [], + args: [], + }, + { + command: 'netstat', + description: 'Display routing information or network connections', + examples: 'netstat\r\nnetstat -nr', + internal_only: false, + runnable: true, + sub_commands: [], + args: [ + { + id: 699, + created_at: '2020-09-22T00:52:52Z', + updated_at: '2020-09-22T00:52:52Z', + script_id: 108, + arg_type: 'flag', + data_type: 'string', + requires_value: false, + arg_name: 'nr', + description: 'Flag to show routing information', + default_value: '', + required: false, + sequence: 1, + options: null, + encoding: '', + command_level: 'non-destructive', + }, + ], + }, + { + command: 'ps', + description: 'Display process information', + examples: 'ps', + internal_only: false, + runnable: true, + sub_commands: [], + args: [], + }, + { + command: 'pwd', + description: 'Prints present working directory', + examples: '', + internal_only: false, + runnable: true, + sub_commands: [], + args: [], + }, + { + command: 'users', + description: 'Get details about local users', + examples: + 'users\r\n List details about all local users\r\nusers foo\r\n List details about local user "foo"', + internal_only: false, + runnable: true, + sub_commands: [], + args: [ + { + id: 719, + created_at: '2023-03-15T00:28:54Z', + updated_at: '2023-03-15T00:28:54Z', + script_id: 117, + arg_type: 'arg', + data_type: 'string', + requires_value: false, + arg_name: 'UserName', + description: 'Username to filter results', + default_value: '', + required: false, + sequence: 1, + options: null, + encoding: '', + command_level: 'non-destructive', + }, + ], + }, + ], + existing_aid_sessions: 1, + created_at: '2024-09-12T07:22:55.684322249Z', + pwd: '/', + offline_queued: false, + }, + ], + errors: null, + }; +}; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/refresh_rtr_session_route.ts b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/refresh_rtr_session_route.ts new file mode 100644 index 0000000000000..77908faf55246 --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/refresh_rtr_session_route.ts @@ -0,0 +1,392 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { buildCrowdstrikeRoutePath, TEST_AGENT_ID, TEST_SESSION_ID } from './utils'; +import type { ExternalEdrServerEmulatorRouteHandlerMethod } from '../../../external_edr_server_emulator.types'; +import type { EmulatorServerRouteDefinition } from '../../../lib/emulator_server.types'; + +export const refreshRTRSessionRoute = (): EmulatorServerRouteDefinition => { + return { + path: buildCrowdstrikeRoutePath('/real-time-response/entities/refresh-session/v1'), + method: 'POST', + handler: refreshRTRSessionHandler, + }; +}; +// requestBody: +// { +// "device_id": "xxxxxx", +// "queue_offline": false +// } + +// @ts-expect-error - example of invalid agent id error +const refreshRTRSessionInvalidAgentIdError = async () => { + return { + meta: { + query_time: 0.244284399, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + batch_id: '', + resources: { + [TEST_AGENT_ID]: { + session_id: '', + complete: false, + stdout: '', + stderr: '', + aid: TEST_AGENT_ID, + errors: [ + { + code: 500, + message: 'uuid: incorrect UUID length 47 in string "wrongAgentId"', + }, + ], + query_time: 0, + offline_queued: false, + }, + }, + errors: [ + { + code: 404, + message: 'no successful hosts initialized on RTR', + }, + ], + }; +}; + +const refreshRTRSessionHandler: ExternalEdrServerEmulatorRouteHandlerMethod<{}, {}> = async () => { + return { + meta: { + query_time: 0.098432609, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + resources: [ + { + session_id: TEST_SESSION_ID, + scripts: [ + { + command: 'cat', + description: 'Read a file from disk and display as ASCII', + examples: 'cat foo.txt\r\ncat -n foo.txt\r\ncat -t foo.txt\r\ncat -t -n foo.txt', + internal_only: false, + runnable: true, + sub_commands: [], + args: [ + { + id: 671, + created_at: '2020-09-22T00:54:20Z', + updated_at: '2020-09-22T00:54:20Z', + script_id: 94, + arg_type: 'arg', + data_type: 'string', + requires_value: false, + arg_name: 'Path', + description: 'path to cat', + default_value: '', + required: true, + sequence: 1, + options: null, + encoding: '', + command_level: 'non-destructive', + }, + { + id: 672, + created_at: '2020-09-22T00:54:20Z', + updated_at: '2020-09-22T00:54:20Z', + script_id: 94, + arg_type: 'flag', + data_type: 'string', + requires_value: false, + arg_name: 'n', + description: 'Number the output lines starting from 1', + default_value: '', + required: false, + sequence: 2, + options: null, + encoding: '', + command_level: 'non-destructive', + }, + { + id: 673, + created_at: '2020-09-22T00:54:20Z', + updated_at: '2020-09-22T00:54:20Z', + script_id: 94, + arg_type: 'flag', + data_type: 'string', + requires_value: false, + arg_name: 't', + description: "Display non-printing characters, and display tab characters as `^I'.", + default_value: '', + required: false, + sequence: 3, + options: null, + encoding: '', + command_level: 'non-destructive', + }, + ], + }, + { + command: 'cd', + description: 'Change the current working directory', + examples: '', + internal_only: false, + runnable: true, + sub_commands: [], + args: [ + { + id: 674, + created_at: '2020-09-22T00:54:07Z', + updated_at: '2020-09-22T00:54:07Z', + script_id: 95, + arg_type: 'arg', + data_type: 'string', + requires_value: false, + arg_name: 'Path', + description: 'path', + default_value: '', + required: true, + sequence: 1, + options: null, + encoding: '', + command_level: 'non-destructive', + }, + ], + }, + { + command: 'env', + description: 'Print out the environment', + examples: 'env', + internal_only: false, + runnable: true, + sub_commands: [], + args: [], + }, + { + command: 'filehash', + description: 'Generate the MD5, SHA1, and SHA256 hashes of a file', + examples: 'filehash /tmp/test', + internal_only: false, + runnable: true, + sub_commands: [], + args: [ + { + id: 680, + created_at: '2020-09-22T00:53:36Z', + updated_at: '2020-09-22T00:53:36Z', + script_id: 100, + arg_type: 'arg', + data_type: 'string', + requires_value: false, + arg_name: 'Path', + description: 'File to hash', + default_value: '', + required: true, + sequence: 1, + options: null, + encoding: '', + command_level: 'non-destructive', + }, + ], + }, + { + command: 'ifconfig', + description: 'Show network configuration information', + examples: 'ifconfig', + internal_only: false, + runnable: true, + sub_commands: [], + args: [], + }, + { + command: 'ls', + description: 'Display the contents of the specified path', + examples: + 'ls\r\nls -l\r\nls -L\r\nls -t\r\nls -l -@\r\nls -R\r\nls -l -R\r\nls -l -t -R -L', + internal_only: false, + runnable: true, + sub_commands: [], + args: [ + { + id: 684, + created_at: '2020-09-22T00:53:14Z', + updated_at: '2020-09-22T00:53:14Z', + script_id: 104, + arg_type: 'arg', + data_type: 'string', + requires_value: false, + arg_name: 'Path', + description: 'Path', + default_value: '.', + required: false, + sequence: 1, + options: null, + encoding: '', + command_level: 'non-destructive', + }, + { + id: 685, + created_at: '2020-09-22T00:53:14Z', + updated_at: '2020-09-22T00:53:14Z', + script_id: 104, + arg_type: 'flag', + data_type: 'string', + requires_value: false, + arg_name: 'l', + description: 'List in long format.', + default_value: '', + required: false, + sequence: 2, + options: null, + encoding: '', + command_level: 'non-destructive', + }, + { + id: 686, + created_at: '2020-09-22T00:53:14Z', + updated_at: '2020-09-22T00:53:14Z', + script_id: 104, + arg_type: 'flag', + data_type: 'string', + requires_value: false, + arg_name: 'L', + description: + 'Follow all symbolic links to final target and list the file or directory the link references rather than the link itself.', + default_value: '', + required: false, + sequence: 3, + options: null, + encoding: '', + command_level: 'non-destructive', + }, + { + id: 687, + created_at: '2020-09-22T00:53:14Z', + updated_at: '2020-09-22T00:53:14Z', + script_id: 104, + arg_type: 'flag', + data_type: 'string', + requires_value: false, + arg_name: 'R', + description: 'Recursively list subdirectories encountered.', + default_value: '', + required: false, + sequence: 4, + options: null, + encoding: '', + command_level: 'non-destructive', + }, + { + id: 688, + created_at: '2020-09-22T00:53:14Z', + updated_at: '2020-09-22T00:53:14Z', + script_id: 104, + arg_type: 'flag', + data_type: 'string', + requires_value: false, + arg_name: 't', + description: + 'Sort by time modified (most recently modified first) before sorting the operands by lexicographical order.', + default_value: '', + required: false, + sequence: 5, + options: null, + encoding: '', + command_level: 'non-destructive', + }, + ], + }, + { + command: 'mount', + description: 'List or mount filesystem volumes', + examples: + 'Executable by all RTR roles:\r\nmount\r\nExecutable by privileged RTR users only:\r\nmount -t=nfs "host:/exports/filesystem" "/mnt/filesystem"\r\n Mount the NFS filesystem located at "/exports/filesystem" on "host" to the local destination "/mnt/filesystem"\r\nmount -t=smbfs "//user:password@host/filesystem" "/mnt/mountpoint"\r\n Mount the SMB "/filesystem" on "host" as "user" with "password" to "/mnt/mountpoint"\r\nmount -t=smbfs -o=nobrowse "//user:password@host/filesystem" "/mnt/mountpoint"\r\n Mount the SMB "/filesystem" with option "nobrowse" on "host" as "user" with "password" to "/mnt/mountpoint"', + internal_only: false, + runnable: true, + sub_commands: [], + args: [], + }, + { + command: 'netstat', + description: 'Display routing information or network connections', + examples: 'netstat\r\nnetstat -nr', + internal_only: false, + runnable: true, + sub_commands: [], + args: [ + { + id: 699, + created_at: '2020-09-22T00:52:52Z', + updated_at: '2020-09-22T00:52:52Z', + script_id: 108, + arg_type: 'flag', + data_type: 'string', + requires_value: false, + arg_name: 'nr', + description: 'Flag to show routing information', + default_value: '', + required: false, + sequence: 1, + options: null, + encoding: '', + command_level: 'non-destructive', + }, + ], + }, + { + command: 'ps', + description: 'Display process information', + examples: 'ps', + internal_only: false, + runnable: true, + sub_commands: [], + args: [], + }, + { + command: 'pwd', + description: 'Prints present working directory', + examples: '', + internal_only: false, + runnable: true, + sub_commands: [], + args: [], + }, + { + command: 'users', + description: 'Get details about local users', + examples: + 'users\r\n List details about all local users\r\nusers foo\r\n List details about local user "foo"', + internal_only: false, + runnable: true, + sub_commands: [], + args: [ + { + id: 719, + created_at: '2023-03-15T00:28:54Z', + updated_at: '2023-03-15T00:28:54Z', + script_id: 117, + arg_type: 'arg', + data_type: 'string', + requires_value: false, + arg_name: 'UserName', + description: 'Username to filter results', + default_value: '', + required: false, + sequence: 1, + options: null, + encoding: '', + command_level: 'non-destructive', + }, + ], + }, + ], + existing_aid_sessions: 1, + created_at: '2024-09-13T08:52:22.671935129Z', + offline_queued: false, + }, + ], + errors: [], + }; +}; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/rtr_admin_route.ts b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/rtr_admin_route.ts new file mode 100644 index 0000000000000..373c6b960948e --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/rtr_admin_route.ts @@ -0,0 +1,264 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { buildCrowdstrikeRoutePath, TEST_AGENT_ID, TEST_SESSION_ID } from './utils'; +import type { ExternalEdrServerEmulatorRouteHandlerMethod } from '../../../external_edr_server_emulator.types'; +import type { EmulatorServerRouteDefinition } from '../../../lib/emulator_server.types'; + +export const rtrAdminCommandRoute = (): EmulatorServerRouteDefinition => { + return { + // we use admin command to run run `runscript` and access `cloudFiles` and `custom scripts` + path: buildCrowdstrikeRoutePath('/real-time-response/combined/batch-admin-command/v1'), + method: 'POST', + handler: rtrAdminCommandHandler, + }; +}; + +// @ts-expect-error - example of error while executing cat command +const rtrAdminCommandExampleError = async () => { + return { + meta: { + query_time: 0.913513625, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + combined: { + resources: { + [TEST_AGENT_ID]: { + session_id: TEST_SESSION_ID, + task_id: 'xxx', + complete: true, + stdout: '', + stderr: 'cat: test.xt: No such file or directory', + base_command: 'cat', + aid: TEST_AGENT_ID, + errors: [], + query_time: 0.912058582, + offline_queued: false, + }, + }, + }, + errors: [], + }; +}; + +// @ts-expect-error - example of inactive rtr session error +const rtrAdminCommandInactiveSessionError = async () => { + return { + meta: { + query_time: 0.02078217, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + combined: { + resources: { + [TEST_AGENT_ID]: { + session_id: '', + complete: false, + stdout: '', + stderr: '', + aid: TEST_AGENT_ID, + errors: [ + { + code: 50007, + message: 'could not get session', + }, + ], + query_time: 0, + offline_queued: false, + }, + }, + }, + errors: [], + }; +}; + +// "command_string": "runscript -CloudFile=\"your_script_name\"", - file not existing +// @ts-expect-error - example of wrong cloud file error +const rtrAdminCommandWrongCloudFileExampleError = async () => { + return { + meta: { + query_time: 0.034585269, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + combined: { + resources: { + [TEST_AGENT_ID]: { + session_id: '', + complete: false, + stdout: '', + stderr: '', + aid: TEST_AGENT_ID, + errors: [ + { + code: 40412, + message: 'The file your_script_name could not be found', + }, + ], + query_time: 0, + offline_queued: false, + }, + }, + }, + errors: [], + }; +}; + +// wrong command eg. asd "command_string": "runscript -Raw=```asd```", +// @ts-expect-error - example of invalid command error +const rtrAdminCommandInvalidCommandError: ExternalEdrServerEmulatorRouteHandlerMethod< + {}, + {} +> = async () => { + return { + meta: { + query_time: 1.959748222, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + combined: { + resources: { + [TEST_AGENT_ID]: { + session_id: TEST_SESSION_ID, + task_id: 'xxx', + complete: true, + stdout: '', + stderr: '/bin/bash: line 1: asd: command not found', + base_command: 'runscript', + aid: TEST_AGENT_ID, + errors: [], + query_time: 1.95571987, + offline_queued: false, + }, + }, + }, + errors: [], + }; +}; + +const rtrAdminCommandHandler: ExternalEdrServerEmulatorRouteHandlerMethod<{}, {}> = async () => { + return { + meta: { + query_time: 0.945570286, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + combined: { + resources: { + [TEST_AGENT_ID]: { + session_id: TEST_SESSION_ID, + task_id: 'xxx', + complete: true, + stdout: + 'archive\nbackup\nbin\nboot\ndev\netc\nhome\nlib\nlost+found\nmedia\nmnt\nopt\nproc\nroot\nrun\nsbin\nsnap\nsrv\nsys\ntmp\nusr\nvar', + stderr: '', + base_command: 'runscript', + aid: TEST_AGENT_ID, + errors: [], + query_time: 0.941080011, + offline_queued: false, + }, + }, + }, + errors: [], + }; +}; +// runscript -CloudFile='test1' (customScript name) - when script is not accessible - eg. private +// @ts-expect-error - example of private script error +const rtrAdminCommandCustomScriptNotFoundError: ExternalEdrServerEmulatorRouteHandlerMethod< + {}, + {} +> = async () => { + return { + meta: { + query_time: 0.01495486, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + combined: { + resources: { + [TEST_AGENT_ID]: { + session_id: '', + complete: false, + stdout: '', + stderr: '', + aid: TEST_AGENT_ID, + errors: [ + { + code: 40412, + message: 'The file test1 could not be found', + }, + ], + query_time: 0, + offline_queued: false, + }, + }, + }, + errors: [], + }; +}; +// @ts-expect-error - example of error while executing put on a file that already exists on host +const rtrCommandPutFileAlreadyExistError: ExternalEdrServerEmulatorRouteHandlerMethod< + {}, + {} +> = async () => { + return { + meta: { + query_time: 7.897133656, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + combined: { + resources: { + [TEST_AGENT_ID]: { + session_id: TEST_SESSION_ID, + task_id: 'xxx', + complete: true, + stdout: '', + stderr: 'put: Destination already exists.', + base_command: 'DOWNLOAD_RENAME', + aid: TEST_AGENT_ID, + errors: [], + query_time: 7.8957342520000005, + offline_queued: false, + }, + }, + }, + errors: [], + }; +}; + +// @ts-expect-error - example of success response while executing put on a file +const rtrAdminCommandPutSuccessResponse: ExternalEdrServerEmulatorRouteHandlerMethod< + {}, + {} +> = async () => { + return { + meta: { + query_time: 5.497466448, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + combined: { + resources: { + [TEST_AGENT_ID]: { + session_id: TEST_SESSION_ID, + task_id: 'xxx', + complete: true, + stdout: '', + stderr: '', + base_command: 'DOWNLOAD_RENAME', + aid: TEST_AGENT_ID, + errors: [], + query_time: 5.496508269, + offline_queued: false, + }, + }, + }, + errors: [], + }; +}; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/rtr_command_route.ts b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/rtr_command_route.ts new file mode 100644 index 0000000000000..60f3c7dd19ca0 --- /dev/null +++ b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/rtr_command_route.ts @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { + buildCrowdstrikeRoutePath, + TEST_AGENT_ID, + TEST_CLOUD_REQUEST_ID, + TEST_SESSION_ID, +} from './utils'; +import type { ExternalEdrServerEmulatorRouteHandlerMethod } from '../../../external_edr_server_emulator.types'; +import type { EmulatorServerRouteDefinition } from '../../../lib/emulator_server.types'; + +export const rtrCommandRoute = (): EmulatorServerRouteDefinition => { + return { + path: buildCrowdstrikeRoutePath('/real-time-response/entities/command/v1'), + method: 'POST', + handler: rtrCommandHandler, + }; +}; +// { +// "base_command": "ls", +// "command_string": "ls", +// "session_id": TEST_SESSION_ID, +// "device_id": TEST_AGENT_ID, +// "persist": true +// } + +// @ts-expect-error - example of error response while executing cat command +const rtrCommandExampleError = async () => { + return { + meta: { + query_time: 0.913513625, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + combined: { + resources: { + [TEST_AGENT_ID]: { + session_id: TEST_SESSION_ID, + task_id: 'xxx', + complete: true, + stdout: '', + stderr: 'cat: test.xt: No such file or directory', + base_command: 'cat', + aid: TEST_AGENT_ID, + errors: [], + query_time: 0.912058582, + offline_queued: false, + }, + }, + }, + errors: [], + }; +}; + +// @ts-expect-error - invalid command +const rtrCommandInvalidCommandError = async () => { + return { + meta: { + query_time: 0.101208469, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + combined: { + resources: { + [TEST_AGENT_ID]: { + session_id: '', + complete: false, + stdout: '', + stderr: '', + aid: TEST_AGENT_ID, + errors: [ + { + code: 40007, + message: 'Command not found', + }, + ], + query_time: 0, + offline_queued: false, + }, + }, + }, + errors: [], + }; +}; + +// @ts-expect-error - invalid session +const rtrCommandInvalidSessionError = async () => { + return { + meta: { + query_time: 0.02078217, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + combined: { + resources: { + [TEST_AGENT_ID]: { + session_id: '', + complete: false, + stdout: '', + stderr: '', + aid: TEST_AGENT_ID, + errors: [ + { + code: 50007, + message: 'could not get session', + }, + ], + query_time: 0, + offline_queued: false, + }, + }, + }, + errors: [], + }; +}; + +const rtrCommandHandler: ExternalEdrServerEmulatorRouteHandlerMethod<{}, {}> = async () => { + return { + meta: { + query_time: 0.274908106, + powered_by: 'empower-api', + trace_id: 'xxx', + }, + resources: [ + { + session_id: TEST_SESSION_ID, + cloud_request_id: TEST_CLOUD_REQUEST_ID, + queued_command_offline: false, + }, + ], + errors: null, + }; +}; diff --git a/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/utils.ts b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/utils.ts index ab8ed1ecd57b6..cdaeb004e213b 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/utils.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/api_emulator/emulator_plugins/crowdstrike/routes/utils.ts @@ -4,6 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ + export const buildCrowdstrikeRoutePath = (path: string): string => { if (!path.startsWith('/')) { throw new Error(`'path' must start with '/'!`); @@ -11,3 +12,9 @@ export const buildCrowdstrikeRoutePath = (path: string): string => { return path; }; + +export const TEST_CID_ID = 'test-cid-id-123456789'; +export const TEST_AGENT_ID = 'test-agent-id-123456789'; +export const TEST_SESSION_ID = 'test-session-id-123456789'; +export const TEST_BATCH_ID = 'test-batch-id-123456789'; +export const TEST_CLOUD_REQUEST_ID = 'test-cloud-request-id-123456789'; From cfcb6e8f4ddf078ccdcec2a03788e39c6e4dfdef Mon Sep 17 00:00:00 2001 From: Konrad Szwarc Date: Tue, 17 Sep 2024 10:18:09 +0200 Subject: [PATCH 03/58] [EDR Workflows][Osquery] Fix last results pack value (#192678) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The “Last Results” tab in the active pack details is displaying incorrect date-time values. This is due to a mismatch in data format. The component responsible for formatting the date-time values expects a string, but it is currently receiving an array with a single string inside. Before: ![Screenshot 2024-09-12 at 10 33 39](https://github.com/user-attachments/assets/844fd699-a086-44d8-aa0b-a06e65e5aa60) After: ![Screenshot 2024-09-12 at 12 40 04](https://github.com/user-attachments/assets/bcce9b69-58b8-438d-89e8-c78a3922845c) --- .../packs/pack_queries_status_table.tsx | 40 +++++++++---------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/x-pack/plugins/osquery/public/packs/pack_queries_status_table.tsx b/x-pack/plugins/osquery/public/packs/pack_queries_status_table.tsx index 53d55aef7475c..d43721a61b6d2 100644 --- a/x-pack/plugins/osquery/public/packs/pack_queries_status_table.tsx +++ b/x-pack/plugins/osquery/public/packs/pack_queries_status_table.tsx @@ -187,7 +187,7 @@ const ViewResultsInLensActionComponent: React.FC { + (event: React.MouseEvent) => { event.preventDefault(); if (logsDataView?.id) { @@ -372,33 +372,31 @@ const ScheduledQueryLastResults: React.FC = ({ interval, }); + const timestamp = useMemo(() => { + const dateTime = lastResultsData?.['@timestamp']; + if (!dateTime) return undefined; + + return Array.isArray(dateTime) ? dateTime[0] : dateTime; + }, [lastResultsData]); + if (isLoading) { return ; } - if (!lastResultsData) { - return <>{'-'}; - } - return ( - {lastResultsData?.['@timestamp'] ? ( + {timestamp ? ( - {' '} - + {' '} + } >
- +
) : ( @@ -584,7 +582,7 @@ const PackQueriesStatusTableComponent: React.FC = ( Record> >({}); - const renderQueryColumn = useCallback((query: string, item: any) => { + const renderQueryColumn = useCallback((query: string, item: PackQueryFormData) => { const singleLine = removeMultilines(query); const content = singleLine.length > 55 ? `${singleLine.substring(0, 55)}...` : singleLine; @@ -618,7 +616,7 @@ const PackQueriesStatusTableComponent: React.FC = ( ); const renderLastResultsColumn = useCallback( - (item: any) => ( + (item: PackQueryFormData) => ( = ( [packName] ); const renderDocsColumn = useCallback( - (item: any) => ( + (item: PackQueryFormData) => ( ), [packName] ); const renderAgentsColumn = useCallback( - (item: any) => ( + (item: PackQueryFormData) => ( ), [packName] ); const renderErrorsColumn = useCallback( - (item: any) => ( + (item: PackQueryFormData) => ( = ( ); const renderDiscoverResultsAction = useCallback( - (item: any) => , + (item: PackQueryFormData) => , [packName] ); const renderLensResultsAction = useCallback( - (item: any) => , + (item: PackQueryFormData) => , [packName] ); From 73fa7a7d8071a7b54e177646294987bff9b6eafb Mon Sep 17 00:00:00 2001 From: Liam Thompson <32779855+leemthompo@users.noreply.github.com> Date: Tue, 17 Sep 2024 09:37:42 +0100 Subject: [PATCH 04/58] [ES3] Update serverless console doclink (#193116) - **404**: https://www.elastic.co/docs/current/serverless/elasticsearch/dev-tools-console - **Correct URL**: https://www.elastic.co/docs/current/serverless/devtools/run-api-requests-in-the-console --- packages/kbn-doc-links/src/get_doc_links.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-doc-links/src/get_doc_links.ts b/packages/kbn-doc-links/src/get_doc_links.ts index 3ad5d271bde47..84005ce3cf492 100644 --- a/packages/kbn-doc-links/src/get_doc_links.ts +++ b/packages/kbn-doc-links/src/get_doc_links.ts @@ -90,7 +90,7 @@ export const getDocLinks = ({ kibanaBranch, buildFlavor }: GetDocLinkOptions): D }, console: { guide: `${KIBANA_DOCS}console-kibana.html`, - serverlessGuide: `${SERVERLESS_ELASTICSEARCH_DOCS}dev-tools-console`, + serverlessGuide: `${SERVERLESS_DOCS}devtools/run-api-requests-in-the-console`, }, dashboard: { guide: `${KIBANA_DOCS}dashboard.html`, From 9c3561d97fcb7f12248bad9b18969dec9dbb819f Mon Sep 17 00:00:00 2001 From: Richa Talwar <102972658+ritalwar@users.noreply.github.com> Date: Tue, 17 Sep 2024 14:25:57 +0530 Subject: [PATCH 05/58] Update autodetect.sh to detect additional integrations in a generic way. (#191383) ## Summary Relates: https://github.com/elastic/observability-dev/issues/3826 Update `detect_known_integrations()` in `auto_detect.sh` for generic detection using `integrations.conf` and added more integrations for auto detect. --- .../auto_detect/auto_detect_panel.tsx | 17 +- .../auto_detect/use_onboarding_flow.tsx | 9 + .../public/application/shared/logo_icon.tsx | 16 + .../public/assets/apache_tomcat.svg | 107 ++++++ .../public/assets/auto_detect.sh | 308 ++++++++++-------- .../public/assets/haproxy.svg | 197 +++++++++++ .../public/assets/integrations.conf | 86 +++++ .../public/assets/kafka.svg | 3 + .../public/assets/mongodb.svg | 10 + .../public/assets/mysql.svg | 6 + .../public/assets/postgresql.svg | 8 + .../public/assets/rabbitmq.svg | 3 + .../public/assets/redis.svg | 1 + 13 files changed, 628 insertions(+), 143 deletions(-) create mode 100644 x-pack/plugins/observability_solution/observability_onboarding/public/assets/apache_tomcat.svg create mode 100644 x-pack/plugins/observability_solution/observability_onboarding/public/assets/haproxy.svg create mode 100644 x-pack/plugins/observability_solution/observability_onboarding/public/assets/integrations.conf create mode 100644 x-pack/plugins/observability_solution/observability_onboarding/public/assets/kafka.svg create mode 100644 x-pack/plugins/observability_solution/observability_onboarding/public/assets/mongodb.svg create mode 100644 x-pack/plugins/observability_solution/observability_onboarding/public/assets/mysql.svg create mode 100644 x-pack/plugins/observability_solution/observability_onboarding/public/assets/postgresql.svg create mode 100644 x-pack/plugins/observability_solution/observability_onboarding/public/assets/rabbitmq.svg create mode 100644 x-pack/plugins/observability_solution/observability_onboarding/public/assets/redis.svg diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/auto_detect/auto_detect_panel.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/auto_detect/auto_detect_panel.tsx index c0bdb89609412..d7070c0d945dd 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/auto_detect/auto_detect_panel.tsx +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/auto_detect/auto_detect_panel.tsx @@ -93,7 +93,22 @@ export const AutoDetectPanel: FunctionComponent = () => { - {['Apache', 'Docker', 'Nginx', 'System', 'Custom .log files'].map((item) => ( + {[ + 'Apache', + 'Docker', + 'Nginx', + 'System', + 'MySQL', + 'PostgreSQL', + 'Redis', + 'HAProxy', + 'Kafka', + 'RabbitMQ', + 'Prometheus', + 'Tomcat', + 'MongoDB', + 'Custom .log files', + ].map((item) => ( {item} diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/auto_detect/use_onboarding_flow.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/auto_detect/use_onboarding_flow.tsx index 9a2c88517cffd..7824e8ddd59a2 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/auto_detect/use_onboarding_flow.tsx +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/auto_detect/use_onboarding_flow.tsx @@ -19,6 +19,15 @@ export const DASHBOARDS = { 'docker-AV4REOpp5NkDleZmzKkE': { type: 'metrics' }, 'nginx-55a9e6e0-a29e-11e7-928f-5dbe6f6f5519': { type: 'logs' }, 'system-79ffd6e0-faa0-11e6-947f-177f697178b8': { type: 'metrics' }, + 'mysql-Logs-MySQL-Dashboard': { type: 'logs' }, + 'postgresql-158be870-87f4-11e7-ad9c-db80de0bf8d3': { type: 'logs' }, + 'redis-7fea2930-478e-11e7-b1f0-cb29bac6bf8b': { type: 'logs' }, + 'haproxy-3560d580-aa34-11e8-9c06-877f0445e3e0': { type: 'logs' }, + 'rabbitmq-AV4YobKIge1VCbKU_qVo': { type: 'metrics' }, + 'kafka-943caca0-87ee-11e7-ad9c-db80de0bf8d3': { type: 'logs' }, + 'apache_tomcat-8fd54a20-1f0d-11ee-9d6b-bb41d08322c8': { type: 'logs' }, + 'mongodb-abcf35b0-0a82-11e8-bffe-ff7d4f68cf94': { type: 'logs' }, + 'prometheus-c181a040-3d96-11ed-b624-b12467b8df74': { type: 'metrics' }, }; export function useOnboardingFlow() { diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/application/shared/logo_icon.tsx b/x-pack/plugins/observability_solution/observability_onboarding/public/application/shared/logo_icon.tsx index 0b49eebb1f94f..f498996f53d20 100644 --- a/x-pack/plugins/observability_solution/observability_onboarding/public/application/shared/logo_icon.tsx +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/application/shared/logo_icon.tsx @@ -24,6 +24,14 @@ export type SupportedLogo = | 'apache' | 'system' | 'opentelemetry' + | 'mysql' + | 'postgresql' + | 'redis' + | 'haproxy' + | 'rabbitmq' + | 'kafka' + | 'mongodb' + | 'apache_tomcat' | 'firehose'; export function isSupportedLogo(logo: string): logo is SupportedLogo { @@ -41,6 +49,14 @@ export function isSupportedLogo(logo: string): logo is SupportedLogo { 'system', 'apache', 'opentelemetry', + 'mysql', + 'postgresql', + 'redis', + 'haproxy', + 'rabbitmq', + 'kafka', + 'mongodb', + 'apache_tomcat', ].includes(logo); } diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/apache_tomcat.svg b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/apache_tomcat.svg new file mode 100644 index 0000000000000..410a468872e17 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/apache_tomcat.svg @@ -0,0 +1,107 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh index 8218ab77b9be0..30d093cf515df 100755 --- a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh @@ -10,15 +10,15 @@ if [ -z "${BASH_VERSION:-}" ]; then fi if ! command -v curl >/dev/null 2>&1; then - fail "curl is required to run this script" + fail "curl is required to run this script" fi # Check if the `lsof` command exists in PATH, if not use `/usr/sbin/lsof` if possible LSOF_PATH="" if command -v lsof >/dev/null 2>&1; then - LSOF_PATH=$(command -v lsof) + LSOF_PATH=$(command -v lsof) elif command -v /usr/sbin/lsof >/dev/null 2>&1; then - LSOF_PATH="/usr/sbin/lsof" + LSOF_PATH="/usr/sbin/lsof" fi install_api_key_encoded="" @@ -28,60 +28,60 @@ onboarding_flow_id="" elastic_agent_version="" help() { - echo "Usage: sudo ./auto-detect.sh " - echo "" - echo "Arguments:" - echo " --install-key= Base64 Encoded API key that has priviledges to install integrations." - echo " --ingest-key= Base64 Encoded API key that has priviledges to ingest data." - echo " --kibana-url= Kibana API endpoint." - echo " --id= Onboarding flow ID." - echo " --ea-version= Elastic Agent version." - exit 1 + echo "Usage: sudo ./auto-detect.sh " + echo "" + echo "Arguments:" + echo " --install-key= Base64 Encoded API key that has priviledges to install integrations." + echo " --ingest-key= Base64 Encoded API key that has priviledges to ingest data." + echo " --kibana-url= Kibana API endpoint." + echo " --id= Onboarding flow ID." + echo " --ea-version= Elastic Agent version." + exit 1 } ensure_argument() { - if [ -z "$1" ]; then - echo "Error: Missing value for $2." - help - fi + if [ -z "$1" ]; then + echo "Error: Missing value for $2." + help + fi } if [ "$EUID" -ne 0 ]; then - echo "Error: This script must be run as root." - help + echo "Error: This script must be run as root." + help fi # Parse command line arguments for i in "$@"; do - case $i in - --install-key=*) - shift - install_api_key_encoded="${i#*=}" - ;; - --ingest-key=*) - shift - ingest_api_key_encoded="${i#*=}" - ;; - --kibana-url=*) - shift - kibana_api_endpoint="${i#*=}" - ;; - --id=*) - shift - onboarding_flow_id="${i#*=}" - ;; - --ea-version=*) - shift - elastic_agent_version="${i#*=}" - ;; - --help) - help - ;; - *) - echo "Unknown option: $i" - help - ;; - esac + case $i in + --install-key=*) + shift + install_api_key_encoded="${i#*=}" + ;; + --ingest-key=*) + shift + ingest_api_key_encoded="${i#*=}" + ;; + --kibana-url=*) + shift + kibana_api_endpoint="${i#*=}" + ;; + --id=*) + shift + onboarding_flow_id="${i#*=}" + ;; + --ea-version=*) + shift + elastic_agent_version="${i#*=}" + ;; + --help) + help + ;; + *) + echo "Unknown option: $i" + help + ;; + esac done ensure_argument "$install_api_key_encoded" "--install-key" @@ -92,6 +92,7 @@ ensure_argument "$elastic_agent_version" "--ea-version" known_integrations_list_string="" selected_known_integrations_array=() +detected_patterns=() selected_known_integrations_tsv_string="" unknown_log_file_path_list_string="" unknown_log_file_pattern_list_string="" @@ -102,6 +103,8 @@ custom_log_file_path_list_tsv_string="" elastic_agent_artifact_name="" elastic_agent_config_path="/opt/Elastic/Agent/elastic-agent.yml" elastic_agent_tmp_config_path="/tmp/elastic-agent-config.tar" +integration_names=() +integration_titles=() OS="$(uname)" ARCH="$(uname -m)" @@ -173,7 +176,7 @@ extract_elastic_agent() { } install_elastic_agent() { - "./${elastic_agent_artifact_name}/elastic-agent" install -f -n > /dev/null + "./${elastic_agent_artifact_name}/elastic-agent" install -f -n >/dev/null if [ "$?" -eq 0 ]; then printf "\e[1;32m✓\e[0m %s\n" "Elastic Agent installed to $(dirname "$elastic_agent_config_path")" @@ -187,11 +190,11 @@ install_elastic_agent() { wait_for_elastic_agent_status() { local MAX_RETRIES=10 local i=0 - elastic-agent status > /dev/null 2>&1 + elastic-agent status >/dev/null 2>&1 local ELASTIC_AGENT_STATUS_EXIT_CODE="$?" while [ "$ELASTIC_AGENT_STATUS_EXIT_CODE" -ne 0 ] && [ $i -le $MAX_RETRIES ]; do sleep 1 - elastic-agent status > /dev/null 2>&1 + elastic-agent status >/dev/null 2>&1 ELASTIC_AGENT_STATUS_EXIT_CODE="$?" ((i++)) done @@ -221,7 +224,7 @@ ensure_elastic_agent_healthy() { backup_elastic_agent_config() { if [ -f "$elastic_agent_config_path" ]; then - echo -e "\nExisting config found at $elastic_agent_config_path"; + echo -e "\nExisting config found at $elastic_agent_config_path" printf "\n\e[1;36m?\e[0m \e[1m%s\e[0m \e[2m%s\e[0m" "Create backup and continue installation?" "[Y/n] (default: Yes): " read confirmation_reply @@ -286,13 +289,13 @@ apply_elastic_agent_config() { local decoded_ingest_api_key=$(echo "$ingest_api_key_encoded" | base64 -d) # Verify that the downloaded archive contains the expected `elastic-agent.yml` file - tar --list --file "$elastic_agent_tmp_config_path" --include 'elastic-agent.yml' > /dev/null && \ - # Remove existing config file including `inputs.d` directory - rm -rf "$elastic_agent_config_path" "$(dirname "$elastic_agent_config_path")/inputs.d" && \ - # Extract new config files from downloaded archive - tar --extract --file "$elastic_agent_tmp_config_path" --include 'elastic-agent.yml' --include 'inputs.d/*.yml' --directory "$(dirname "$elastic_agent_config_path")" && \ - # Replace placeholder with the Ingest API key - sed -i '' "s/\${API_KEY}/$decoded_ingest_api_key/" "$elastic_agent_config_path" + tar --list --file "$elastic_agent_tmp_config_path" --include 'elastic-agent.yml' >/dev/null && + # Remove existing config file including `inputs.d` directory + rm -rf "$elastic_agent_config_path" "$(dirname "$elastic_agent_config_path")/inputs.d" && + # Extract new config files from downloaded archive + tar --extract --file "$elastic_agent_tmp_config_path" --include 'elastic-agent.yml' --include 'inputs.d/*.yml' --directory "$(dirname "$elastic_agent_config_path")" && + # Replace placeholder with the Ingest API key + sed -i '' "s/\${API_KEY}/$decoded_ingest_api_key/" "$elastic_agent_config_path" if [ "$?" -eq 0 ]; then printf "\e[1;32m✓\e[0m %s\n" "Config written to:" tar --list --file "$elastic_agent_tmp_config_path" --include 'elastic-agent.yml' --include 'inputs.d/*.yml' | while read -r file; do @@ -313,30 +316,29 @@ read_open_log_file_list() { "^\/Users\/.+?\/Library\/Containers" "^\/Users\/.+?\/Library\/Caches" "^\/private" - # Excluding all patterns that correspond to known integrations - # that we are detecting separately - "^\/var\/log\/nginx" - "^\/var\/log\/apache2" - "^\/var\/log\/httpd" - "^\/var\/lib\/docker\/containers" - "^\/var\/log\/syslog" - "^\/var\/log\/auth.log" - "^\/var\/log\/system.log" - "^\/var\/log\/messages" - "^\/var\/log\/secure" + # Exclude previous installation logs "\/opt\/Elastic\/Agent\/" "\/Library\/Elastic\/Agent\/" ) + # Excluding all patterns that correspond to known integrations + # that we are detecting separately + for pattern in "${detected_patterns[@]}"; do + exclude_patterns+=("$pattern") + done + local list=$("$LSOF_PATH" -Fn / | grep "^n.*\.log$" | cut -c2- | sort -u) # Filtering by the exclude patterns while IFS= read -r line; do - if ! grep -qE "$(IFS="|"; echo "${exclude_patterns[*]}")" <<< "$line"; then - unknown_log_file_path_list_string+="$line\n" + if ! grep -qE "$( + IFS="|" + echo "${exclude_patterns[*]}" + )" <<<"$line"; then + unknown_log_file_path_list_string+="$line\n" fi - done <<< "$list" + done <<<"$list" } detect_known_integrations() { @@ -344,60 +346,82 @@ detect_known_integrations() { # Even when there is no system logs on the host, # System integration will still be able to to collect metrics. known_integrations_list_string+="system"$'\n' + integrations_config_url="${kibana_api_endpoint}/plugins/observabilityOnboarding/assets/integrations.conf" - local nginx_patterns=( - "/var/log/nginx/access.log*" - "/var/log/nginx/error.log*" - ) + integrations_config=$(curl "${integrations_config_url}" --silent --fail) + local integration="" + local patterns=() + + # Debug: Check if the config file exists + if [[ -z "$integrations_config" ]]; then + echo "Failed to retrieve config file" + exit 1 + fi - for pattern in "${nginx_patterns[@]}"; do - if compgen -G "$pattern" > /dev/null; then - known_integrations_list_string+="nginx"$'\n' - break + while IFS= read -r line; do + + # Skip comments and empty lines + if [[ $line =~ ^\s*# || -z $line ]]; then + continue fi - done - local apache_patterns=( - "/var/log/apache2/access.log*" - "/var/log/apache2/other_vhosts_access.log*" - "/var/log/apache2/error.log*" - "/var/log/httpd/access_log*" - "/var/log/httpd/error_log*" - ) + # Process section headers + if [[ $line =~ ^\[([a-zA-Z0-9_]+)\] ]]; then + # If we were processing a previous section, check patterns for the previous integration + if [[ -n "$integration" && ${#patterns[@]} -gt 0 ]]; then + for pattern in "${patterns[@]}"; do + pattern=$(echo "$pattern" | xargs) # Trim leading/trailing spaces + if compgen -G "$pattern" >/dev/null; then + known_integrations_list_string+="$integration"$'\n' + detected_patterns+=("${patterns[@]}") + break + fi + done + fi - for pattern in "${apache_patterns[@]}"; do - if compgen -G "$pattern" > /dev/null; then - known_integrations_list_string+="apache"$'\n' - break + # Start a new section + integration="${BASH_REMATCH[1]}" + patterns=() + continue fi - done - if [ -S /var/run/docker.sock ]; then - known_integrations_list_string+="docker"$'\n' - elif compgen -G "/var/lib/docker/containers/*/*-json.log" > /dev/null; then - known_integrations_list_string+="docker"$'\n' + # Process patterns + if [[ $line =~ ^patterns= ]]; then + # Capture patterns by trimming spaces and handling multi-line patterns + IFS=$'\n' read -r -d '' -a patterns <<<"${line#patterns=}" + patterns=($(echo "${patterns[@]}" | xargs)) # Trim leading/trailing spaces + elif [[ $line =~ ^title=.*$ ]]; then + # Capture titles + integration_titles+=("${line#title=}") + integration_names+=("$integration") + elif [[ -n "$integration" && -n "$line" ]]; then + # Capture multi-line patterns if not directly following "patterns=" + patterns+=("$(echo "$line" | xargs)") # Trim leading/trailing spaces + fi + done <<< "$integrations_config" + + # Check patterns for the last section + if [[ -n "$integration" && ${#patterns[@]} -gt 0 ]]; then + for pattern in "${patterns[@]}"; do + pattern=$(echo "$pattern" | xargs) # Trim leading/trailing spaces + if compgen -G "$pattern" >/dev/null; then + known_integrations_list_string+="$integration"$'\n' + detected_patterns+=("${patterns[@]}") + break + fi + done fi } known_integration_title() { local integration=$1 - case $integration in - "nginx") - echo "Nginx Logs" - ;; - "apache") - echo "Apache Logs" - ;; - "docker") - echo "Docker Container Logs" - ;; - "system") - echo "System Logs And Metrics" - ;; - *) - echo "Unknown" - ;; - esac + for i in "${!integration_names[@]}"; do + if [[ "${integration_names[$i]}" == "$integration" ]]; then + echo "${integration_titles[$i]}" + return + fi + done + echo "Unknown" } build_unknown_log_file_patterns() { @@ -407,7 +431,7 @@ build_unknown_log_file_patterns() { fi unknown_log_file_pattern_list_string+="$(dirname "$log_file_path")/*.log\n" - done <<< "$(echo -e "$unknown_log_file_path_list_string")" + done <<<"$(echo -e "$unknown_log_file_path_list_string")" unknown_log_file_pattern_list_string=$(echo -e "$unknown_log_file_pattern_list_string" | sort -u) } @@ -421,14 +445,14 @@ function select_list() { continue fi known_integrations_options+=("$line") - done <<< "$known_integrations_list_string" + done <<<"$known_integrations_list_string" while IFS= read -r line; do if [[ -z "$line" ]]; then continue fi unknown_logs_options+=("$line") - done <<< "$unknown_log_file_pattern_list_string" + done <<<"$unknown_log_file_pattern_list_string" local options=("${known_integrations_options[@]}" "${unknown_logs_options[@]}") @@ -448,13 +472,13 @@ function select_list() { printf "\n\e[1;36m?\e[0m \e[1m%s\e[0m \e[2m%s\e[0m\n" "Exclude logs by listing their index numbers" "(e.g. 1, 2, 3). Press Enter to skip." read exclude_index_list_string - IFS=', ' read -r -a exclude_index_list_array <<< "$exclude_index_list_string" + IFS=', ' read -r -a exclude_index_list_array <<<"$exclude_index_list_string" for index in "${!options[@]}"; do local is_excluded=0 for excluded_index in "${exclude_index_list_array[@]}"; do if [[ "$index" -eq "$((excluded_index - 1))" ]]; then - is_excluded=1 + is_excluded=1 fi done @@ -481,7 +505,7 @@ function select_list() { printf "\e[1;36m?\e[0m \e[1m%s\e[0m \e[2m%s\e[0m\n" "List any additional logs you'd like to ingest" "(e.g. /path1/*.log, /path2/*.log). Press Enter to skip." read custom_log_file_path_list_string - IFS=', ' read -r -a custom_log_file_path_list_array <<< "$custom_log_file_path_list_string" + IFS=', ' read -r -a custom_log_file_path_list_array <<<"$custom_log_file_path_list_string" echo -e "\nYou've selected these logs for ingestion:" for item in "${selected_known_integrations_array[@]}"; do @@ -506,30 +530,30 @@ function select_list() { } generate_custom_integration_name() { - local path_pattern="$1" - local dir_path - local name_parts=() - local name + local path_pattern="$1" + local dir_path + local name_parts=() + local name - dir_path=$(dirname "$path_pattern") - IFS='/' read -r -a dir_array <<< "$dir_path" + dir_path=$(dirname "$path_pattern") + IFS='/' read -r -a dir_array <<<"$dir_path" - # Get the last up to 4 parts of the path - for (( i=${#dir_array[@]}-1, count=0; i>=0 && count<4; i--, count++ )); do - name_parts=("${dir_array[$i]}" "${name_parts[@]}") - done + # Get the last up to 4 parts of the path + for ((i = ${#dir_array[@]} - 1, count = 0; i >= 0 && count < 4; i--, count++)); do + name_parts=("${dir_array[$i]}" "${name_parts[@]}") + done - # Join the parts into a single string with underscores - name=$(printf "%s_" "${name_parts[@]}") - name="${name#_}" # Remove leading underscore - name="${name%_}" # Remove trailing underscore + # Join the parts into a single string with underscores + name=$(printf "%s_" "${name_parts[@]}") + name="${name#_}" # Remove leading underscore + name="${name%_}" # Remove trailing underscore - # Replace special characters with underscores - name="${name// /_}" - name="${name//-/_}" - name="${name//./_}" + # Replace special characters with underscores + name="${name// /_}" + name="${name//-/_}" + name="${name//./_}" - echo "$name" + echo "$name" } printf "\e[1m%s\e[0m\n" "Looking for log files..." @@ -538,10 +562,10 @@ detect_known_integrations # Check if LSOF_PATH is executable if [ -x "$LSOF_PATH" ]; then - read_open_log_file_list - build_unknown_log_file_patterns + read_open_log_file_list + build_unknown_log_file_patterns else - echo -e "\nlsof is required to detect custom log files. Looking for known integrations only." + echo -e "\nlsof is required to detect custom log files. Looking for known integrations only." fi update_step_progress "logs-detect" "complete" diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/haproxy.svg b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/haproxy.svg new file mode 100644 index 0000000000000..f45c35d3434af --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/haproxy.svg @@ -0,0 +1,197 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/integrations.conf b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/integrations.conf new file mode 100644 index 0000000000000..e6455a9170c86 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/integrations.conf @@ -0,0 +1,86 @@ +[system] +title=System Logs And Metrics + +[nginx] +title=Nginx Logs +patterns= + /var/log/nginx/access.log* + /var/log/nginx/error.log* + +[apache] +title=Apache Logs +patterns= + /var/log/apache2/access.log* + /var/log/apache2/other_vhosts_access.log* + /var/log/apache2/error.log* + /var/log/httpd/access_log* + /var/log/httpd/error_log* + +[docker] +title=Docker Container Logs +patterns= + /var/lib/docker/containers/*/*-json.log + /var/run/docker.sock + +[mysql] +title=MySQL Logs +patterns= + /var/log/mysql/*error.log* + /var/log/mysqld.log* + /var/log/mysql/*-slow.log* + /var/lib/mysql/*-slow.log* + +[postgresql] +title=PostgreSQL Logs +patterns= + /var/log/postgresql/postgresql-*-*.log* + /*/postgresql-logs/*.log + /etc/postgresql/*/main/postgresql.conf + /var/log/postgresql/postgresql-*-*.csv* + +[redis] +title=Redis Logs +patterns= + /var/log/redis/redis-server.log* + /etc/redis/redis.conf + +[haproxy] +title=HAProxy Logs +patterns= + /var/log/haproxy.log + /etc/haproxy/haproxy.cfg + +[rabbitmq] +title=RabbitMQ Logs +patterns= + /var/log/rabbitmq/rabbit@*.log + /etc/rabbitmq/rabbitmq.conf + /etc/rabbitmq/rabbitmq.config + +[kafka] +title=Kafka Logs +patterns= + /var/log/kafka/server.log + /etc/kafka/server.properties + /*/logs/controller.log* + /*/logs/server.log* + /*/logs/state-change.log* + /*/logs/kafka-*.log* + +[mongodb] +title=MongoDB Logs +patterns= + /var/log/mongodb/mongod.log + +[apache_tomcat] +title=Apache Tomcat Logs +patterns= + /opt/tomcat/logs/localhost_access_log.*.txt + /opt/tomcat/logs/catalina.*.log + /opt/tomcat/logs/localhost.*.log + +[prometheus] +title=Prometheus Server overview +patterns= + /var/log/prometheus/prometheus.log + /etc/prometheus/prometheus.yml \ No newline at end of file diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/kafka.svg b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/kafka.svg new file mode 100644 index 0000000000000..e88f77cb55b41 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/kafka.svg @@ -0,0 +1,3 @@ + + + diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/mongodb.svg b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/mongodb.svg new file mode 100644 index 0000000000000..1727f81d2f6c9 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/mongodb.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/mysql.svg b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/mysql.svg new file mode 100644 index 0000000000000..cfe6cbb664e7f --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/mysql.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/postgresql.svg b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/postgresql.svg new file mode 100644 index 0000000000000..0306131fcd395 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/postgresql.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/rabbitmq.svg b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/rabbitmq.svg new file mode 100644 index 0000000000000..dabd2a5744cb4 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/rabbitmq.svg @@ -0,0 +1,3 @@ + + + diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/redis.svg b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/redis.svg new file mode 100644 index 0000000000000..1163d1ea52f44 --- /dev/null +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/redis.svg @@ -0,0 +1 @@ + From 7939f60f563af7c3f5402fcc70a7bb3578644e62 Mon Sep 17 00:00:00 2001 From: Jeramy Soucy Date: Tue, 17 Sep 2024 11:03:26 +0200 Subject: [PATCH 06/58] =?UTF-8?q?Upgrade=20express=204.19.2=E2=86=924.21.0?= =?UTF-8?q?=20(#192862)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Upgrades `express` from v4.19.2 to v4.21.0, and `@types/express` from v4.17.13 to v4.17.21. This PR also moves the dev-only dependency `@mswjs/http-middleware` into the `devDependencies` section. --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- package.json | 6 +-- yarn.lock | 148 +++++++++++++++++++++++++++------------------------ 2 files changed, 81 insertions(+), 73 deletions(-) diff --git a/package.json b/package.json index 5b9e8fac1fb31..1a420b7cae9ea 100644 --- a/package.json +++ b/package.json @@ -996,7 +996,6 @@ "@mapbox/mapbox-gl-rtl-text": "0.2.3", "@mapbox/mapbox-gl-supported": "2.0.1", "@mapbox/vector-tile": "1.3.1", - "@mswjs/http-middleware": "^0.10.1", "@opentelemetry/api": "^1.1.0", "@opentelemetry/api-metrics": "^0.31.0", "@opentelemetry/exporter-metrics-otlp-grpc": "^0.34.0", @@ -1464,6 +1463,7 @@ "@kbn/whereis-pkg-cli": "link:packages/kbn-whereis-pkg-cli", "@kbn/yarn-lock-validator": "link:packages/kbn-yarn-lock-validator", "@mapbox/vector-tile": "1.3.1", + "@mswjs/http-middleware": "^0.10.1", "@octokit/rest": "^17.11.2", "@parcel/watcher": "^2.1.0", "@playwright/test": "=1.46.0", @@ -1525,7 +1525,7 @@ "@types/enzyme": "^3.10.12", "@types/eslint": "^8.44.2", "@types/event-stream": "^4.0.5", - "@types/express": "^4.17.13", + "@types/express": "^4.17.21", "@types/extract-zip": "^1.6.2", "@types/faker": "^5.1.5", "@types/fetch-mock": "^7.3.1", @@ -1700,7 +1700,7 @@ "exit-hook": "^2.2.0", "expect": "^29.7.0", "expose-loader": "^0.7.5", - "express": "^4.19.2", + "express": "^4.21.0", "faker": "^5.1.0", "fetch-mock": "^7.3.9", "file-loader": "^4.2.0", diff --git a/yarn.lock b/yarn.lock index 408d93ad5ee99..9b0438134fd4c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10358,22 +10358,23 @@ resolved "https://registry.yarnpkg.com/@types/expect/-/expect-1.20.4.tgz#8288e51737bf7e3ab5d7c77bfa695883745264e5" integrity sha512-Q5Vn3yjTDyCMV50TB6VRIbQNxSE4OmZR86VSbGaNpfUolm0iePBB4KdEEHmxoY5sT2+2DIvXW0rvMDP2nHZ4Mg== -"@types/express-serve-static-core@*", "@types/express-serve-static-core@^4.17.18": - version "4.17.28" - resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.17.28.tgz#c47def9f34ec81dc6328d0b1b5303d1ec98d86b8" - integrity sha512-P1BJAEAW3E2DJUlkgq4tOL3RyMunoWXqbSCygWo5ZIWTjUgN1YnaXWW4VWl/oc8vs/XoYibEGBKP0uZyF4AHig== +"@types/express-serve-static-core@*", "@types/express-serve-static-core@^4.17.33": + version "4.19.5" + resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.19.5.tgz#218064e321126fcf9048d1ca25dd2465da55d9c6" + integrity sha512-y6W03tvrACO72aijJ5uF02FRq5cgDR9lUxddQ8vyF+GvmjJQqbzDcJngEjURc+ZsG31VI3hODNZJ2URj86pzmg== dependencies: "@types/node" "*" "@types/qs" "*" "@types/range-parser" "*" + "@types/send" "*" -"@types/express@*", "@types/express@^4.17.13": - version "4.17.13" - resolved "https://registry.yarnpkg.com/@types/express/-/express-4.17.13.tgz#a76e2995728999bab51a33fabce1d705a3709034" - integrity sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA== +"@types/express@*", "@types/express@^4.17.13", "@types/express@^4.17.21": + version "4.17.21" + resolved "https://registry.npmjs.org/@types/express/-/express-4.17.21.tgz" + integrity sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ== dependencies: "@types/body-parser" "*" - "@types/express-serve-static-core" "^4.17.18" + "@types/express-serve-static-core" "^4.17.33" "@types/qs" "*" "@types/serve-static" "*" @@ -11279,6 +11280,14 @@ resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.8.tgz#8268a8c57a3e4abd25c165ecd36237db7948a55e" integrity sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ== +"@types/send@*": + version "0.17.4" + resolved "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz" + integrity sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA== + dependencies: + "@types/mime" "^1" + "@types/node" "*" + "@types/serve-index@^1.9.1": version "1.9.1" resolved "https://registry.yarnpkg.com/@types/serve-index/-/serve-index-1.9.1.tgz#1b5e85370a192c01ec6cec4735cf2917337a6278" @@ -13409,10 +13418,10 @@ bn.js@^5.0.0, bn.js@^5.2.1: resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-5.2.1.tgz#0bc527a6a0d18d0aa8d5b0538ce4a77dccfa7b70" integrity sha512-eXRvHzWyYPBuB4NBy0cmYQjGitUrtqwbvlzP3G6VFnNRbsZQIxQ10PbKKHt8gZ/HW/D/747aDl+QkDqg3KQLMQ== -body-parser@1.20.2: - version "1.20.2" - resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.2.tgz#6feb0e21c4724d06de7ff38da36dad4f57a747fd" - integrity sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA== +body-parser@1.20.3: + version "1.20.3" + resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.3.tgz#1953431221c6fb5cd63c4b36d53fab0928e548c6" + integrity sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g== dependencies: bytes "3.1.2" content-type "~1.0.5" @@ -13422,7 +13431,7 @@ body-parser@1.20.2: http-errors "2.0.0" iconv-lite "0.4.24" on-finished "2.4.1" - qs "6.11.0" + qs "6.13.0" raw-body "2.5.2" type-is "~1.6.18" unpipe "1.0.0" @@ -16729,6 +16738,11 @@ encodeurl@~1.0.2: resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" integrity sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k= +encodeurl@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-2.0.0.tgz#7b8ea898077d7e409d3ac45474ea38eaf0857a58" + integrity sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg== + end-of-stream@^1.0.0, end-of-stream@^1.1.0, end-of-stream@^1.4.1, end-of-stream@^1.4.4: version "1.4.4" resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" @@ -17765,37 +17779,37 @@ expr-eval@^2.0.2: resolved "https://registry.yarnpkg.com/expr-eval/-/expr-eval-2.0.2.tgz#fa6f044a7b0c93fde830954eb9c5b0f7fbc7e201" integrity sha512-4EMSHGOPSwAfBiibw3ndnP0AvjDWLsMvGOvWEZ2F96IGk0bIVdjQisOHxReSkE13mHcfbuCiXw+G4y0zv6N8Eg== -express@^4.17.1, express@^4.17.3, express@^4.18.2, express@^4.19.2: - version "4.19.2" - resolved "https://registry.yarnpkg.com/express/-/express-4.19.2.tgz#e25437827a3aa7f2a827bc8171bbbb664a356465" - integrity sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q== +express@^4.17.1, express@^4.17.3, express@^4.18.2, express@^4.21.0: + version "4.21.0" + resolved "https://registry.yarnpkg.com/express/-/express-4.21.0.tgz#d57cb706d49623d4ac27833f1cbc466b668eb915" + integrity sha512-VqcNGcj/Id5ZT1LZ/cfihi3ttTn+NJmkli2eZADigjq29qTlWi/hAQ43t/VLPq8+UX06FCEx3ByOYet6ZFblng== dependencies: accepts "~1.3.8" array-flatten "1.1.1" - body-parser "1.20.2" + body-parser "1.20.3" content-disposition "0.5.4" content-type "~1.0.4" cookie "0.6.0" cookie-signature "1.0.6" debug "2.6.9" depd "2.0.0" - encodeurl "~1.0.2" + encodeurl "~2.0.0" escape-html "~1.0.3" etag "~1.8.1" - finalhandler "1.2.0" + finalhandler "1.3.1" fresh "0.5.2" http-errors "2.0.0" - merge-descriptors "1.0.1" + merge-descriptors "1.0.3" methods "~1.1.2" on-finished "2.4.1" parseurl "~1.3.3" - path-to-regexp "0.1.7" + path-to-regexp "0.1.10" proxy-addr "~2.0.7" - qs "6.11.0" + qs "6.13.0" range-parser "~1.2.1" safe-buffer "5.2.1" - send "0.18.0" - serve-static "1.15.0" + send "0.19.0" + serve-static "1.16.2" setprototypeof "1.2.0" statuses "2.0.1" type-is "~1.6.18" @@ -18152,13 +18166,13 @@ fill-range@^7.1.1: dependencies: to-regex-range "^5.0.1" -finalhandler@1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.2.0.tgz#7d23fe5731b207b4640e4fcd00aec1f9207a7b32" - integrity sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg== +finalhandler@1.3.1: + version "1.3.1" + resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.3.1.tgz#0c575f1d1d324ddd1da35ad7ece3df7d19088019" + integrity sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ== dependencies: debug "2.6.9" - encodeurl "~1.0.2" + encodeurl "~2.0.0" escape-html "~1.0.3" on-finished "2.4.1" parseurl "~1.3.3" @@ -18714,7 +18728,7 @@ get-east-asian-width@^1.0.0: resolved "https://registry.yarnpkg.com/get-east-asian-width/-/get-east-asian-width-1.2.0.tgz#5e6ebd9baee6fb8b7b6bd505221065f0cd91f64e" integrity sha512-2nk+7SIVb14QrgXFHcm84tD4bKQz0RxPuMT8Ag5KPOq7J5fEmAg0UbXdTOSHqNuHSU28k55qnceesxXRZGzKWA== -get-intrinsic@^1.0.2, get-intrinsic@^1.1.3, get-intrinsic@^1.2.1, get-intrinsic@^1.2.2, get-intrinsic@^1.2.3, get-intrinsic@^1.2.4: +get-intrinsic@^1.1.3, get-intrinsic@^1.2.1, get-intrinsic@^1.2.2, get-intrinsic@^1.2.3, get-intrinsic@^1.2.4: version "1.2.4" resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.2.4.tgz#e385f5a4b5227d449c3eabbad05494ef0abbeadd" integrity sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ== @@ -23082,10 +23096,10 @@ meow@^9.0.0: type-fest "^0.18.0" yargs-parser "^20.2.3" -merge-descriptors@1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61" - integrity sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E= +merge-descriptors@1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.3.tgz#d80319a65f3c7935351e5cfdac8f9318504dbed5" + integrity sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ== merge-source-map@1.0.4: version "1.0.4" @@ -24436,7 +24450,7 @@ object-identity-map@^1.0.2: dependencies: object.entries "^1.1.0" -object-inspect@^1.13.1, object-inspect@^1.6.0, object-inspect@^1.7.0, object-inspect@^1.9.0: +object-inspect@^1.13.1, object-inspect@^1.6.0, object-inspect@^1.7.0: version "1.13.2" resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.13.2.tgz#dea0088467fb991e67af4058147a24824a3043ff" integrity sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g== @@ -25164,10 +25178,10 @@ path-scurry@^1.11.1: lru-cache "^10.2.0" minipass "^5.0.0 || ^6.0.2 || ^7.0.0" -path-to-regexp@0.1.7: - version "0.1.7" - resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz" - integrity sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ== +path-to-regexp@0.1.10: + version "0.1.10" + resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.10.tgz#67e9108c5c0551b9e5326064387de4763c4d5f8b" + integrity sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w== path-to-regexp@^1.7.0: version "1.9.0" @@ -26292,19 +26306,12 @@ qs@6.10.4: dependencies: side-channel "^1.0.4" -qs@6.11.0: - version "6.11.0" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.11.0.tgz#fd0d963446f7a65e1367e01abd85429453f0c37a" - integrity sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q== +qs@6.13.0, qs@^6.10.0, qs@^6.11.0, qs@^6.7.0: + version "6.13.0" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.13.0.tgz#6ca3bd58439f7e245655798997787b0d88a51906" + integrity sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg== dependencies: - side-channel "^1.0.4" - -qs@^6.10.0, qs@^6.11.0, qs@^6.7.0: - version "6.11.2" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.11.2.tgz#64bea51f12c1f5da1bc01496f48ffcff7c69d7d9" - integrity sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA== - dependencies: - side-channel "^1.0.4" + side-channel "^1.0.6" query-string@^6.13.2: version "6.13.2" @@ -28412,10 +28419,10 @@ semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.5.0, semve resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== -send@0.18.0: - version "0.18.0" - resolved "https://registry.yarnpkg.com/send/-/send-0.18.0.tgz#670167cc654b05f5aa4a767f9113bb371bc706be" - integrity sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg== +send@0.19.0: + version "0.19.0" + resolved "https://registry.yarnpkg.com/send/-/send-0.19.0.tgz#bbc5a388c8ea6c048967049dbeac0e4a3f09d7f8" + integrity sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw== dependencies: debug "2.6.9" depd "2.0.0" @@ -28483,15 +28490,15 @@ serve-index@^1.9.1: mime-types "~2.1.17" parseurl "~1.3.2" -serve-static@1.15.0: - version "1.15.0" - resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.15.0.tgz#faaef08cffe0a1a62f60cad0c4e513cff0ac9540" - integrity sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g== +serve-static@1.16.2: + version "1.16.2" + resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.16.2.tgz#b6a5343da47f6bdd2673848bf45754941e803296" + integrity sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw== dependencies: - encodeurl "~1.0.2" + encodeurl "~2.0.0" escape-html "~1.0.3" parseurl "~1.3.3" - send "0.18.0" + send "0.19.0" set-blocking@^2.0.0: version "2.0.0" @@ -28691,14 +28698,15 @@ should@^13.2.1: should-type-adaptors "^1.0.1" should-util "^1.0.0" -side-channel@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.4.tgz#efce5c8fdc104ee751b25c58d4290011fa5ea2cf" - integrity sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw== +side-channel@^1.0.4, side-channel@^1.0.6: + version "1.0.6" + resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.6.tgz#abd25fb7cd24baf45466406b1096b7831c9215f2" + integrity sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA== dependencies: - call-bind "^1.0.0" - get-intrinsic "^1.0.2" - object-inspect "^1.9.0" + call-bind "^1.0.7" + es-errors "^1.3.0" + get-intrinsic "^1.2.4" + object-inspect "^1.13.1" signal-exit@^3.0.0, signal-exit@^3.0.2, signal-exit@^3.0.3, signal-exit@^3.0.7: version "3.0.7" From 6b24114b4298266d7c4d73186983f9b73704a7a2 Mon Sep 17 00:00:00 2001 From: Cristina Amico Date: Tue, 17 Sep 2024 11:35:00 +0200 Subject: [PATCH 07/58] [Fleet] Remove deprecated Symantec package from install_all_packages_job (#193029) ## Summary [Job](https://buildkite.com/elastic/kibana-fleet-packages/builds/1022) `install_all_packages` is been failing for several days because of `symantec-0.1.3` being deprecated. Querying ``` GET kbn:/api/fleet/epm/packages?prerelease=true ``` it returns: ``` "name": "symantec", "title": "Symantec", "version": "0.1.3", "release": "experimental", "description": "Deprecated. Use a specific Symantec package instead.", ``` I'm not sure when this deprecation was announced but I'm skipping this package from the script so the job should hopefully return green. EDIT: package got deprecated back in 2022: https://github.com/elastic/package-storage/pull/3254 however last week we merged a [change](https://github.com/elastic/kibana/pull/192040/files#diff-292c3f307d3d0d341a361d12416d04609a3f525be268242c2a06be06fd8d5810R188) to temporarily remove kibana version checks when querying EPR, so this package started appearing. In fact in previous successful runs we didn't attempt installing this package at all. Co-authored-by: Elastic Machine --- x-pack/test/fleet_packages/tests/install_all.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/x-pack/test/fleet_packages/tests/install_all.ts b/x-pack/test/fleet_packages/tests/install_all.ts index d58c9a0333b84..e835f200ed490 100644 --- a/x-pack/test/fleet_packages/tests/install_all.ts +++ b/x-pack/test/fleet_packages/tests/install_all.ts @@ -14,6 +14,11 @@ import { FtrProviderContext } from '../../api_integration/ftr_provider_context'; +const DEPRECATED_PACKAGES = [ + 'zscaler', // deprecated: https://github.com/elastic/integrations/issues/4947 + 'symantec', +]; + export default function (providerContext: FtrProviderContext) { const { getService } = providerContext; const supertest = getService('supertest'); @@ -69,8 +74,9 @@ export default function (providerContext: FtrProviderContext) { .expect(200); const allResults = []; for (const pkg of packages) { - // skip deprecated failing package https://github.com/elastic/integrations/issues/4947 - if (pkg.name === 'zscaler') continue; + // skip deprecated failing packages + if (DEPRECATED_PACKAGES.includes(pkg.name)) continue; + const res = await installPackage(pkg.name, pkg.version); allResults.push(res); if (res.success) { From 808212e97e413216655aaa9e755c671656decb46 Mon Sep 17 00:00:00 2001 From: Bena Kansara <69037875+benakansara@users.noreply.github.com> Date: Tue, 17 Sep 2024 11:50:09 +0200 Subject: [PATCH 08/58] [RCA] [Recent events] Create API endpoint to get events (#192947) Closes https://github.com/elastic/observability-dev/issues/3924 Closes https://github.com/elastic/observability-dev/issues/3927 This PR introduces an events API (`/api/observability/events`) that will fetch - - All the "point in time" annotations from` observability-annotations` index. This includes both manual and auto (e.g. service deployment) annotations - The annotations will be filtered with supported source fields (host.name, service.name, slo.id, slo.instanceId) when specified as `filter` - Alerts that newly triggered on same source in given time range. The source needs to be specified as `filter`, when no filter is specified all alerts triggered in given time range will be returned ### Testing - Create annotations (APM service deployment annotations and annotations using Observability UI) - Generate some alerts - API call should return annotations and alerts, example API requests - `http://localhost:5601/kibana/api/observability/events?rangeFrom=2024-09-01T19:53:20.243Z&rangeTo=2024-09-19T19:53:20.243Z&filter={"annotation.type":"deployment"}` - `http://localhost:5601/kibana/api/observability/events?rangeFrom=2024-09-01T19:53:20.243Z&rangeTo=2024-09-19T19:53:20.243Z&filter={"slo.id":"*"}` - `http://localhost:5601/kibana/api/observability/events?rangeFrom=2024-09-01T19:53:20.243Z&rangeTo=2024-09-19T19:53:20.243Z&filter={"host.name":"host-0"}` - `http://localhost:5601/kibana/api/observability/events?rangeFrom=2024-09-01T19:53:20.243Z&rangeTo=2024-09-19T19:53:20.243Z` --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../src/rest_specs/event.ts | 18 +++ .../src/rest_specs/get_events.ts | 31 +++++ .../src/rest_specs/index.ts | 4 + .../src/schema/event.ts | 51 ++++++++ .../src/schema/index.ts | 1 + .../investigate_app/kibana.jsonc | 3 + ...investigate_app_server_route_repository.ts | 32 +++++ .../investigate_app/server/routes/types.ts | 2 + .../server/services/get_alerts_client.ts | 49 +++++++ .../server/services/get_events.ts | 123 ++++++++++++++++++ .../investigate_app/server/types.ts | 15 ++- .../investigate_app/tsconfig.json | 3 + .../observability/common/annotations.ts | 2 + .../annotations/create_annotations_client.ts | 45 ++++--- 14 files changed, 359 insertions(+), 20 deletions(-) create mode 100644 packages/kbn-investigation-shared/src/rest_specs/event.ts create mode 100644 packages/kbn-investigation-shared/src/rest_specs/get_events.ts create mode 100644 packages/kbn-investigation-shared/src/schema/event.ts create mode 100644 x-pack/plugins/observability_solution/investigate_app/server/services/get_alerts_client.ts create mode 100644 x-pack/plugins/observability_solution/investigate_app/server/services/get_events.ts diff --git a/packages/kbn-investigation-shared/src/rest_specs/event.ts b/packages/kbn-investigation-shared/src/rest_specs/event.ts new file mode 100644 index 0000000000000..df2f3941ad332 --- /dev/null +++ b/packages/kbn-investigation-shared/src/rest_specs/event.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { z } from '@kbn/zod'; +import { eventSchema } from '../schema'; + +const eventResponseSchema = eventSchema; + +type EventResponse = z.output; + +export { eventResponseSchema }; +export type { EventResponse }; diff --git a/packages/kbn-investigation-shared/src/rest_specs/get_events.ts b/packages/kbn-investigation-shared/src/rest_specs/get_events.ts new file mode 100644 index 0000000000000..064a75fab1562 --- /dev/null +++ b/packages/kbn-investigation-shared/src/rest_specs/get_events.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { z } from '@kbn/zod'; +import { eventResponseSchema } from './event'; + +const getEventsParamsSchema = z + .object({ + query: z + .object({ + rangeFrom: z.string(), + rangeTo: z.string(), + filter: z.string(), + }) + .partial(), + }) + .partial(); + +const getEventsResponseSchema = z.array(eventResponseSchema); + +type GetEventsParams = z.infer; +type GetEventsResponse = z.output; + +export { getEventsParamsSchema, getEventsResponseSchema }; +export type { GetEventsParams, GetEventsResponse }; diff --git a/packages/kbn-investigation-shared/src/rest_specs/index.ts b/packages/kbn-investigation-shared/src/rest_specs/index.ts index c00ec5035765e..42bec32041af4 100644 --- a/packages/kbn-investigation-shared/src/rest_specs/index.ts +++ b/packages/kbn-investigation-shared/src/rest_specs/index.ts @@ -25,6 +25,8 @@ export type * from './investigation_note'; export type * from './update'; export type * from './update_item'; export type * from './update_note'; +export type * from './event'; +export type * from './get_events'; export * from './create'; export * from './create_item'; @@ -44,3 +46,5 @@ export * from './investigation_note'; export * from './update'; export * from './update_item'; export * from './update_note'; +export * from './event'; +export * from './get_events'; diff --git a/packages/kbn-investigation-shared/src/schema/event.ts b/packages/kbn-investigation-shared/src/schema/event.ts new file mode 100644 index 0000000000000..c954a0de13fb3 --- /dev/null +++ b/packages/kbn-investigation-shared/src/schema/event.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { z } from '@kbn/zod'; + +const eventTypeSchema = z.union([ + z.literal('annotation'), + z.literal('alert'), + z.literal('error_rate'), + z.literal('latency'), + z.literal('anomaly'), +]); + +const annotationEventSchema = z.object({ + eventType: z.literal('annotation'), + annotationType: z.string().optional(), +}); + +const alertStatusSchema = z.union([ + z.literal('active'), + z.literal('flapping'), + z.literal('recovered'), + z.literal('untracked'), +]); + +const alertEventSchema = z.object({ + eventType: z.literal('alert'), + alertStatus: alertStatusSchema, +}); + +const sourceSchema = z.record(z.string(), z.any()); + +const eventSchema = z.intersection( + z.object({ + id: z.string(), + title: z.string(), + description: z.string(), + timestamp: z.number(), + eventType: eventTypeSchema, + source: sourceSchema.optional(), + }), + z.discriminatedUnion('eventType', [annotationEventSchema, alertEventSchema]) +); + +export { eventSchema }; diff --git a/packages/kbn-investigation-shared/src/schema/index.ts b/packages/kbn-investigation-shared/src/schema/index.ts index 7491ecce76cc2..f65fe9baf1f6f 100644 --- a/packages/kbn-investigation-shared/src/schema/index.ts +++ b/packages/kbn-investigation-shared/src/schema/index.ts @@ -11,5 +11,6 @@ export * from './investigation'; export * from './investigation_item'; export * from './investigation_note'; export * from './origin'; +export * from './event'; export type * from './investigation'; diff --git a/x-pack/plugins/observability_solution/investigate_app/kibana.jsonc b/x-pack/plugins/observability_solution/investigate_app/kibana.jsonc index ecfe77b5a0584..2cc904dafac05 100644 --- a/x-pack/plugins/observability_solution/investigate_app/kibana.jsonc +++ b/x-pack/plugins/observability_solution/investigate_app/kibana.jsonc @@ -19,6 +19,9 @@ "datasetQuality", "unifiedSearch", "security", + "observability", + "licensing", + "ruleRegistry" ], "requiredBundles": [ "esql", diff --git a/x-pack/plugins/observability_solution/investigate_app/server/routes/get_global_investigate_app_server_route_repository.ts b/x-pack/plugins/observability_solution/investigate_app/server/routes/get_global_investigate_app_server_route_repository.ts index b0ecd89275914..195fbdb234360 100644 --- a/x-pack/plugins/observability_solution/investigate_app/server/routes/get_global_investigate_app_server_route_repository.ts +++ b/x-pack/plugins/observability_solution/investigate_app/server/routes/get_global_investigate_app_server_route_repository.ts @@ -21,7 +21,10 @@ import { updateInvestigationItemParamsSchema, updateInvestigationNoteParamsSchema, updateInvestigationParamsSchema, + getEventsParamsSchema, + GetEventsResponse, } from '@kbn/investigation-shared'; +import { ScopedAnnotationsClient } from '@kbn/observability-plugin/server'; import { createInvestigation } from '../services/create_investigation'; import { createInvestigationItem } from '../services/create_investigation_item'; import { createInvestigationNote } from '../services/create_investigation_note'; @@ -35,6 +38,8 @@ import { getInvestigationItems } from '../services/get_investigation_items'; import { getInvestigationNotes } from '../services/get_investigation_notes'; import { investigationRepositoryFactory } from '../services/investigation_repository'; import { updateInvestigation } from '../services/update_investigation'; +import { getAlertEvents, getAnnotationEvents } from '../services/get_events'; +import { AlertsClient, getAlertsClient } from '../services/get_alerts_client'; import { updateInvestigationItem } from '../services/update_investigation_item'; import { updateInvestigationNote } from '../services/update_investigation_note'; import { createInvestigateAppServerRoute } from './create_investigate_app_server_route'; @@ -313,6 +318,32 @@ const deleteInvestigationItemRoute = createInvestigateAppServerRoute({ }, }); +const getEventsRoute = createInvestigateAppServerRoute({ + endpoint: 'GET /api/observability/events 2023-10-31', + options: { + tags: [], + }, + params: getEventsParamsSchema, + handler: async ({ params, context, request, plugins }) => { + const annotationsClient: ScopedAnnotationsClient | undefined = + await plugins.observability.setup.getScopedAnnotationsClient(context, request); + const alertsClient: AlertsClient = await getAlertsClient({ plugins, request }); + const events: GetEventsResponse = []; + + if (annotationsClient) { + const annotationEvents = await getAnnotationEvents(params?.query ?? {}, annotationsClient); + events.push(...annotationEvents); + } + + if (alertsClient) { + const alertEvents = await getAlertEvents(params?.query ?? {}, alertsClient); + events.push(...alertEvents); + } + + return events; + }, +}); + export function getGlobalInvestigateAppServerRouteRepository() { return { ...createInvestigationRoute, @@ -328,6 +359,7 @@ export function getGlobalInvestigateAppServerRouteRepository() { ...deleteInvestigationItemRoute, ...updateInvestigationItemRoute, ...getInvestigationItemsRoute, + ...getEventsRoute, ...getAllInvestigationStatsRoute, ...getAllInvestigationTagsRoute, }; diff --git a/x-pack/plugins/observability_solution/investigate_app/server/routes/types.ts b/x-pack/plugins/observability_solution/investigate_app/server/routes/types.ts index 2e882296adff0..afb022cdc9b7f 100644 --- a/x-pack/plugins/observability_solution/investigate_app/server/routes/types.ts +++ b/x-pack/plugins/observability_solution/investigate_app/server/routes/types.ts @@ -14,6 +14,7 @@ import type { SavedObjectsClientContract, } from '@kbn/core/server'; import type { Logger } from '@kbn/logging'; +import { LicensingApiRequestHandlerContext } from '@kbn/licensing-plugin/server'; import type { InvestigateAppSetupDependencies, InvestigateAppStartDependencies } from '../types'; export type InvestigateAppRequestHandlerContext = Omit< @@ -33,6 +34,7 @@ export type InvestigateAppRequestHandlerContext = Omit< }; coreStart: CoreStart; }>; + licensing: Promise; }; export interface InvestigateAppRouteHandlerResources { diff --git a/x-pack/plugins/observability_solution/investigate_app/server/services/get_alerts_client.ts b/x-pack/plugins/observability_solution/investigate_app/server/services/get_alerts_client.ts new file mode 100644 index 0000000000000..bf1070307742a --- /dev/null +++ b/x-pack/plugins/observability_solution/investigate_app/server/services/get_alerts_client.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isEmpty } from 'lodash'; +import { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types'; +import { ParsedTechnicalFields } from '@kbn/rule-registry-plugin/common'; +import { InvestigateAppRouteHandlerResources } from '../routes/types'; + +export type AlertsClient = Awaited>; + +export async function getAlertsClient({ + plugins, + request, +}: Pick) { + const ruleRegistryPluginStart = await plugins.ruleRegistry.start(); + const alertsClient = await ruleRegistryPluginStart.getRacClientWithRequest(request); + const alertsIndices = await alertsClient.getAuthorizedAlertsIndices([ + 'logs', + 'infrastructure', + 'apm', + 'slo', + 'uptime', + 'observability', + ]); + + if (!alertsIndices || isEmpty(alertsIndices)) { + throw Error('No alert indices exist'); + } + + type RequiredParams = ESSearchRequest & { + size: number; + track_total_hits: boolean | number; + }; + + return { + search( + searchParams: TParams + ): Promise> { + return alertsClient.find({ + ...searchParams, + index: alertsIndices.join(','), + }) as Promise; + }, + }; +} diff --git a/x-pack/plugins/observability_solution/investigate_app/server/services/get_events.ts b/x-pack/plugins/observability_solution/investigate_app/server/services/get_events.ts new file mode 100644 index 0000000000000..52eeea7a4cbcc --- /dev/null +++ b/x-pack/plugins/observability_solution/investigate_app/server/services/get_events.ts @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import datemath from '@elastic/datemath'; +import { estypes } from '@elastic/elasticsearch'; +import { + GetEventsParams, + GetEventsResponse, + getEventsResponseSchema, +} from '@kbn/investigation-shared'; +import { ScopedAnnotationsClient } from '@kbn/observability-plugin/server'; +import { + ALERT_REASON, + ALERT_RULE_CATEGORY, + ALERT_START, + ALERT_STATUS, + ALERT_UUID, +} from '@kbn/rule-data-utils'; +import { AlertsClient } from './get_alerts_client'; + +export function rangeQuery( + start: number, + end: number, + field = '@timestamp' +): estypes.QueryDslQueryContainer[] { + return [ + { + range: { + [field]: { + gte: start, + lte: end, + format: 'epoch_millis', + }, + }, + }, + ]; +} + +export async function getAnnotationEvents( + params: GetEventsParams, + annotationsClient: ScopedAnnotationsClient +): Promise { + const response = await annotationsClient.find({ + start: params?.rangeFrom, + end: params?.rangeTo, + filter: params?.filter, + size: 100, + }); + + // we will return only "point_in_time" annotations + const events = response.items + .filter((item) => !item.event?.end) + .map((item) => { + const hostName = item.host?.name; + const serviceName = item.service?.name; + const serviceVersion = item.service?.version; + const sloId = item.slo?.id; + const sloInstanceId = item.slo?.instanceId; + + return { + id: item.id, + title: item.annotation.title, + description: item.message, + timestamp: new Date(item['@timestamp']).getTime(), + eventType: 'annotation', + annotationType: item.annotation.type, + source: { + ...(hostName ? { 'host.name': hostName } : undefined), + ...(serviceName ? { 'service.name': serviceName } : undefined), + ...(serviceVersion ? { 'service.version': serviceVersion } : undefined), + ...(sloId ? { 'slo.id': sloId } : undefined), + ...(sloInstanceId ? { 'slo.instanceId': sloInstanceId } : undefined), + }, + }; + }); + + return getEventsResponseSchema.parse(events); +} + +export async function getAlertEvents( + params: GetEventsParams, + alertsClient: AlertsClient +): Promise { + const startInMs = datemath.parse(params?.rangeFrom ?? 'now-15m')!.valueOf(); + const endInMs = datemath.parse(params?.rangeTo ?? 'now')!.valueOf(); + const filterJSON = params?.filter ? JSON.parse(params.filter) : {}; + + const body = { + size: 100, + track_total_hits: false, + query: { + bool: { + filter: [ + ...rangeQuery(startInMs, endInMs, ALERT_START), + ...Object.keys(filterJSON).map((filterKey) => ({ + term: { [filterKey]: filterJSON[filterKey] }, + })), + ], + }, + }, + }; + + const response = await alertsClient.search(body); + + const events = response.hits.hits.map((hit) => { + const _source = hit._source; + + return { + id: _source[ALERT_UUID], + title: `${_source[ALERT_RULE_CATEGORY]} breached`, + description: _source[ALERT_REASON], + timestamp: new Date(_source['@timestamp']).getTime(), + eventType: 'alert', + alertStatus: _source[ALERT_STATUS], + }; + }); + + return getEventsResponseSchema.parse(events); +} diff --git a/x-pack/plugins/observability_solution/investigate_app/server/types.ts b/x-pack/plugins/observability_solution/investigate_app/server/types.ts index 6fa1196b23b74..fa4db6ccfcb05 100644 --- a/x-pack/plugins/observability_solution/investigate_app/server/types.ts +++ b/x-pack/plugins/observability_solution/investigate_app/server/types.ts @@ -5,13 +5,24 @@ * 2.0. */ +import { ObservabilityPluginSetup } from '@kbn/observability-plugin/server'; +import { + RuleRegistryPluginSetupContract, + RuleRegistryPluginStartContract, +} from '@kbn/rule-registry-plugin/server'; + /* eslint-disable @typescript-eslint/no-empty-interface*/ export interface ConfigSchema {} -export interface InvestigateAppSetupDependencies {} +export interface InvestigateAppSetupDependencies { + observability: ObservabilityPluginSetup; + ruleRegistry: RuleRegistryPluginSetupContract; +} -export interface InvestigateAppStartDependencies {} +export interface InvestigateAppStartDependencies { + ruleRegistry: RuleRegistryPluginStartContract; +} export interface InvestigateAppServerSetup {} diff --git a/x-pack/plugins/observability_solution/investigate_app/tsconfig.json b/x-pack/plugins/observability_solution/investigate_app/tsconfig.json index 03651f3530c6d..cd687f2dcfe70 100644 --- a/x-pack/plugins/observability_solution/investigate_app/tsconfig.json +++ b/x-pack/plugins/observability_solution/investigate_app/tsconfig.json @@ -58,5 +58,8 @@ "@kbn/lens-embeddable-utils", "@kbn/i18n-react", "@kbn/zod", + "@kbn/observability-plugin", + "@kbn/licensing-plugin", + "@kbn/rule-data-utils", ], } diff --git a/x-pack/plugins/observability_solution/observability/common/annotations.ts b/x-pack/plugins/observability_solution/observability/common/annotations.ts index 16c6e11b81e86..874234acc6ced 100644 --- a/x-pack/plugins/observability_solution/observability/common/annotations.ts +++ b/x-pack/plugins/observability_solution/observability/common/annotations.ts @@ -96,6 +96,8 @@ export const findAnnotationRt = t.partial({ sloId: t.string, sloInstanceId: t.string, serviceName: t.string, + filter: t.string, + size: t.number, }); export const updateAnnotationRt = t.intersection([ diff --git a/x-pack/plugins/observability_solution/observability/server/lib/annotations/create_annotations_client.ts b/x-pack/plugins/observability_solution/observability/server/lib/annotations/create_annotations_client.ts index 6cf3b63459827..5bd7395c3ca71 100644 --- a/x-pack/plugins/observability_solution/observability/server/lib/annotations/create_annotations_client.ts +++ b/x-pack/plugins/observability_solution/observability/server/lib/annotations/create_annotations_client.ts @@ -202,7 +202,12 @@ export function createAnnotationsClient(params: { }; }), find: ensureGoldLicense(async (findParams: FindAnnotationParams) => { - const { start, end, sloId, sloInstanceId, serviceName } = findParams ?? {}; + const { start, end, sloId, sloInstanceId, serviceName, filter, size } = findParams ?? {}; + const filterJSON = filter ? JSON.parse(filter) : {}; + + const termsFilter = Object.keys(filterJSON).map((filterKey) => ({ + term: { [filterKey]: filterJSON[filterKey] }, + })); const shouldClauses: QueryDslQueryContainer[] = []; if (sloId || sloInstanceId) { @@ -246,7 +251,7 @@ export function createAnnotationsClient(params: { const result = await esClient.search({ index: readIndex, - size: 10000, + size: size ?? 10000, ignore_unavailable: true, query: { bool: { @@ -259,22 +264,26 @@ export function createAnnotationsClient(params: { }, }, }, - { - bool: { - should: [ - ...(serviceName - ? [ - { - term: { - 'service.name': serviceName, - }, - }, - ] - : []), - ...shouldClauses, - ], - }, - }, + ...(Object.keys(filterJSON).length !== 0 + ? termsFilter + : [ + { + bool: { + should: [ + ...(serviceName + ? [ + { + term: { + 'service.name': serviceName, + }, + }, + ] + : []), + ...shouldClauses, + ], + }, + }, + ]), ], }, }, From 4a2dd8a99370ce8d6678de15dd617565e5bfc5ac Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Tue, 17 Sep 2024 12:36:24 +0200 Subject: [PATCH 09/58] [APM] Empty/null service.environment error fix (#192448) fixes [192380](https://github.com/elastic/kibana/issues/192380) ## Summary Revert the `service.environment` comparison change. Using `equals` can be dangerous if the field is null, resulting in errors. The previous version worked fine, so I decided to revert the change that modified the condition. --------- Co-authored-by: Elastic Machine --- .../service_map/fetch_service_paths_from_trace_ids.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/service_map/fetch_service_paths_from_trace_ids.ts b/x-pack/plugins/observability_solution/apm/server/routes/service_map/fetch_service_paths_from_trace_ids.ts index 70d51a56c6177..5224ed833ff24 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/service_map/fetch_service_paths_from_trace_ids.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/service_map/fetch_service_paths_from_trace_ids.ts @@ -113,6 +113,10 @@ export async function fetchServicePathsFromTraceIds({ const numDocsPerShardAllowed = calculatedDocs > terminateAfter ? terminateAfter : calculatedDocs; + /* + * Any changes to init_script, map_script, combine_script and reduce_script + * must be replicated on https://github.com/elastic/elasticsearch-serverless/blob/main/distribution/archives/src/serverless-default-settings.yml + */ const serviceMapAggs = { service_map: { scripted_metric: { @@ -224,10 +228,9 @@ export async function fetchServicePathsFromTraceIds({ // if the parent has 'span.destination.service.resource' set, and the service is different, we've discovered a service if (parent['span.destination.service.resource'] != null - && !parent['span.destination.service.resource'].equals("") - && (!parent['service.name'].equals(event['service.name']) - || !parent['service.environment'].equals(event['service.environment']) - ) + && parent['span.destination.service.resource'] != "" + && (parent['service.name'] != event['service.name'] + || parent['service.environment'] != event['service.environment']) ) { def parentDestination = getDestination(parent); context.externalToServiceMap.put(parentDestination, service); From 15c752cdc86c1ca5f7d0c30b3305e477cb14e669 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cau=C3=AA=20Marcondes?= <55978943+cauemarcondes@users.noreply.github.com> Date: Tue, 17 Sep 2024 11:38:01 +0100 Subject: [PATCH 10/58] [Inventory] Project restructure to show entities grid (#192991) Inventory plugin restructure. Creating server API to fetch entities and initial data grid load on the page. --- .../inventory/common/entities.ts | 30 +++--- .../public/components/entities_grid/index.tsx | 68 +++++++++++++ .../entity_type_list/index.stories.tsx | 88 ----------------- .../components/entity_type_list/index.tsx | 96 ------------------- .../inventory_page_template/index.tsx | 63 ++---------- .../public/pages/inventory_page/index.tsx | 16 ++++ .../inventory/public/routes/config.tsx | 3 +- .../create_entities_es_client.ts | 80 ++++++++++++++++ .../routes/entities/get_latest_entities.ts | 27 ++++++ .../inventory/server/routes/entities/route.ts | 33 +++---- .../inventory/server/utils/with_apm_span.ts | 7 ++ .../inventory/tsconfig.json | 3 +- 12 files changed, 243 insertions(+), 271 deletions(-) create mode 100644 x-pack/plugins/observability_solution/inventory/public/components/entities_grid/index.tsx delete mode 100644 x-pack/plugins/observability_solution/inventory/public/components/entity_type_list/index.stories.tsx delete mode 100644 x-pack/plugins/observability_solution/inventory/public/components/entity_type_list/index.tsx create mode 100644 x-pack/plugins/observability_solution/inventory/public/pages/inventory_page/index.tsx create mode 100644 x-pack/plugins/observability_solution/inventory/server/lib/create_es_client/create_entities_es_client.ts create mode 100644 x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts create mode 100644 x-pack/plugins/observability_solution/inventory/server/utils/with_apm_span.ts diff --git a/x-pack/plugins/observability_solution/inventory/common/entities.ts b/x-pack/plugins/observability_solution/inventory/common/entities.ts index af0e5c82b978f..d72fa46969b8a 100644 --- a/x-pack/plugins/observability_solution/inventory/common/entities.ts +++ b/x-pack/plugins/observability_solution/inventory/common/entities.ts @@ -5,16 +5,22 @@ * 2.0. */ -export interface EntityTypeDefinition { - type: string; - label: string; - icon: string; - count: number; -} - -export interface EntityDefinition { - type: string; - field: string; - filter?: string; - index: string[]; +export interface LatestEntity { + agent: { + name: string[]; + }; + data_stream: { + type: string[]; + }; + cloud: { + availability_zone: string[]; + }; + entity: { + firstSeenTimestamp: string; + lastSeenTimestamp: string; + type: string; + displayName: string; + id: string; + identityFields: string[]; + }; } diff --git a/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/index.tsx b/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/index.tsx new file mode 100644 index 0000000000000..e689063882c40 --- /dev/null +++ b/x-pack/plugins/observability_solution/inventory/public/components/entities_grid/index.tsx @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { + EuiDataGrid, + EuiDataGridCellValueElementProps, + EuiDataGridColumn, + EuiLoadingSpinner, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { useAbortableAsync } from '@kbn/observability-utils/hooks/use_abortable_async'; +import React, { useState } from 'react'; +import { useKibana } from '../../hooks/use_kibana'; + +const columns: EuiDataGridColumn[] = [ + { + id: 'entityName', + displayAsText: 'Entity name', + }, + { + id: 'entityType', + displayAsText: 'Type', + }, +]; + +export function EntitiesGrid() { + const { + services: { inventoryAPIClient }, + } = useKibana(); + const [visibleColumns, setVisibleColumns] = useState(columns.map(({ id }) => id)); + const { value = { entities: [] }, loading } = useAbortableAsync( + ({ signal }) => { + return inventoryAPIClient.fetch('GET /internal/inventory/entities', { + signal, + }); + }, + [inventoryAPIClient] + ); + + if (loading) { + return ; + } + + function CellValue({ rowIndex, columnId, setCellProps }: EuiDataGridCellValueElementProps) { + const data = value.entities[rowIndex]; + if (data === undefined) { + return null; + } + + return <>{data.entity.displayName}; + } + + return ( + + ); +} diff --git a/x-pack/plugins/observability_solution/inventory/public/components/entity_type_list/index.stories.tsx b/x-pack/plugins/observability_solution/inventory/public/components/entity_type_list/index.stories.tsx deleted file mode 100644 index 570622406c9ae..0000000000000 --- a/x-pack/plugins/observability_solution/inventory/public/components/entity_type_list/index.stories.tsx +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { Meta, StoryObj } from '@storybook/react'; -import React from 'react'; -import { mergePlainObjects } from '@kbn/investigate-plugin/common'; -import { EntityTypeListBase as Component } from '.'; -import { KibanaReactStorybookDecorator } from '../../../.storybook/storybook_decorator'; - -interface Args { - props: Omit, 'onLockAllClick' | 'onUnlockAllClick'>; -} - -type StoryMeta = Meta; -type Story = StoryObj; - -const meta: StoryMeta = { - component: Component, - title: 'app/Molecules/EntityTypeList', - decorators: [KibanaReactStorybookDecorator], -}; - -export default meta; - -const defaultStory: Story = { - args: { - props: { - definitions: [], - loading: true, - }, - }, - render: function Render(args) { - return ( -
- -
- ); - }, -}; - -export const Default: Story = { - ...defaultStory, - args: { - props: mergePlainObjects(defaultStory.args!.props!, { - loading: false, - definitions: [ - { - icon: 'node', - label: 'Services', - type: 'service', - count: 9, - }, - { - icon: 'pipeNoBreaks', - label: 'Datasets', - type: 'dataset', - count: 11, - }, - ], - }), - }, - name: 'default', -}; - -export const Empty: Story = { - ...defaultStory, - args: { - props: mergePlainObjects(defaultStory.args!.props!, { - definitions: [], - loading: false, - }), - }, - name: 'empty', -}; - -export const Loading: Story = { - ...defaultStory, - args: { - props: mergePlainObjects(defaultStory.args!.props!, { - loading: true, - }), - }, - name: 'loading', -}; diff --git a/x-pack/plugins/observability_solution/inventory/public/components/entity_type_list/index.tsx b/x-pack/plugins/observability_solution/inventory/public/components/entity_type_list/index.tsx deleted file mode 100644 index 47488f23f3252..0000000000000 --- a/x-pack/plugins/observability_solution/inventory/public/components/entity_type_list/index.tsx +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import React from 'react'; -import { useAbortableAsync } from '@kbn/observability-utils/hooks/use_abortable_async'; -import { - EuiBadge, - EuiFlexGroup, - EuiFlexItem, - EuiIcon, - EuiLink, - EuiLoadingSpinner, - EuiText, -} from '@elastic/eui'; -import { useKibana } from '../../hooks/use_kibana'; -import { EntityTypeDefinition } from '../../../common/entities'; -import { useInventoryRouter } from '../../hooks/use_inventory_router'; - -export function EntityTypeListItem({ - href, - icon, - label, - count, -}: { - href: string; - icon: string; - label: string; - count: number; -}) { - return ( - - - - - - - {label} - - - {count} - - - - ); -} - -export function EntityTypeListBase({ - definitions, - loading, - error, -}: { - loading?: boolean; - definitions?: EntityTypeDefinition[]; - error?: Error; -}) { - const router = useInventoryRouter(); - if (loading) { - return ; - } - - return ( - - {definitions?.map((definition) => { - return ( - - ); - })} - - ); -} - -export function EntityTypeList() { - const { - services: { inventoryAPIClient }, - } = useKibana(); - - const { value, loading, error } = useAbortableAsync( - ({ signal }) => { - return inventoryAPIClient.fetch('GET /internal/inventory/entity_types', { - signal, - }); - }, - [inventoryAPIClient] - ); - - return ; -} diff --git a/x-pack/plugins/observability_solution/inventory/public/components/inventory_page_template/index.tsx b/x-pack/plugins/observability_solution/inventory/public/components/inventory_page_template/index.tsx index 386df9a51cae5..4dd8eaf3899ee 100644 --- a/x-pack/plugins/observability_solution/inventory/public/components/inventory_page_template/index.tsx +++ b/x-pack/plugins/observability_solution/inventory/public/components/inventory_page_template/index.tsx @@ -4,13 +4,9 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { EuiFlexGroup, EuiPanel, EuiTitle } from '@elastic/eui'; -import { css } from '@emotion/css'; import { i18n } from '@kbn/i18n'; -import { useTheme } from '@kbn/observability-utils/hooks/use_theme'; import React from 'react'; import { useKibana } from '../../hooks/use_kibana'; -import { EntityTypeList } from '../entity_type_list'; export function InventoryPageTemplate({ children }: { children: React.ReactNode }) { const { @@ -19,60 +15,17 @@ export function InventoryPageTemplate({ children }: { children: React.ReactNode }, } = useKibana(); - const { PageTemplate } = observabilityShared.navigation; - - const theme = useTheme(); + const { PageTemplate: ObservabilityPageTemplate } = observabilityShared.navigation; return ( - - - - - -

- {i18n.translate('xpack.inventory.inventoryPageHeaderLabel', { - defaultMessage: 'Inventory', - })} -

- - - - - - - - - {children} - - - + {children} + ); } diff --git a/x-pack/plugins/observability_solution/inventory/public/pages/inventory_page/index.tsx b/x-pack/plugins/observability_solution/inventory/public/pages/inventory_page/index.tsx new file mode 100644 index 0000000000000..9389fdaca3ea0 --- /dev/null +++ b/x-pack/plugins/observability_solution/inventory/public/pages/inventory_page/index.tsx @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import React from 'react'; +import { EntitiesGrid } from '../../components/entities_grid'; + +export function InventoryPage() { + return ( +
+ +
+ ); +} diff --git a/x-pack/plugins/observability_solution/inventory/public/routes/config.tsx b/x-pack/plugins/observability_solution/inventory/public/routes/config.tsx index 11d9d4836d981..74eeaac220bc2 100644 --- a/x-pack/plugins/observability_solution/inventory/public/routes/config.tsx +++ b/x-pack/plugins/observability_solution/inventory/public/routes/config.tsx @@ -8,6 +8,7 @@ import * as t from 'io-ts'; import { createRouter, Outlet } from '@kbn/typed-react-router-config'; import React from 'react'; import { InventoryPageTemplate } from '../components/inventory_page_template'; +import { InventoryPage } from '../pages/inventory_page'; /** * The array of route definitions to be used when the application @@ -28,7 +29,7 @@ const inventoryRoutes = { }), }, '/': { - element: <>, + element: , }, }, }, diff --git a/x-pack/plugins/observability_solution/inventory/server/lib/create_es_client/create_entities_es_client.ts b/x-pack/plugins/observability_solution/inventory/server/lib/create_es_client/create_entities_es_client.ts new file mode 100644 index 0000000000000..983a4df3e96af --- /dev/null +++ b/x-pack/plugins/observability_solution/inventory/server/lib/create_es_client/create_entities_es_client.ts @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types'; +import type { KibanaRequest } from '@kbn/core/server'; +import { ElasticsearchClient } from '@kbn/core/server'; +import { entitiesAliasPattern, ENTITY_LATEST } from '@kbn/entities-schema'; +import { unwrapEsResponse } from '@kbn/observability-shared-plugin/common/utils/unwrap_es_response'; +// import { withApmSpan } from '../../utils/with_apm_span'; + +const ENTITIES_LATEST_ALIAS = entitiesAliasPattern({ + type: '*', + dataset: ENTITY_LATEST, +}); + +export function cancelEsRequestOnAbort>( + promise: T, + request: KibanaRequest, + controller: AbortController +): T { + const subscription = request.events.aborted$.subscribe(() => { + controller.abort(); + }); + + return promise.finally(() => subscription.unsubscribe()) as T; +} + +export interface EntitiesESClient { + searchLatest( + operationName: string, + searchRequest: TSearchRequest + ): Promise>; +} + +export function createEntitiesESClient({ + request, + esClient, +}: { + request: KibanaRequest; + esClient: ElasticsearchClient; +}) { + function search( + indexName: string, + operationName: string, + searchRequest: TSearchRequest + ): Promise> { + const controller = new AbortController(); + + const promise = // withApmSpan(operationName, () => { + cancelEsRequestOnAbort( + esClient.search( + { ...searchRequest, index: [indexName], ignore_unavailable: true }, + { + signal: controller.signal, + meta: true, + } + ) as unknown as Promise<{ + body: InferSearchResponseOf; + }>, + request, + controller + ); + // }); + // + return unwrapEsResponse(promise); + } + + return { + searchLatest( + operationName: string, + searchRequest: TSearchRequest + ): Promise> { + return search(ENTITIES_LATEST_ALIAS, operationName, searchRequest); + }, + }; +} diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts new file mode 100644 index 0000000000000..4ddcaaf75c9a4 --- /dev/null +++ b/x-pack/plugins/observability_solution/inventory/server/routes/entities/get_latest_entities.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { LatestEntity } from '../../../common/entities'; +import { EntitiesESClient } from '../../lib/create_es_client/create_entities_es_client'; + +const MAX_NUMBER_OF_ENTITIES = 500; + +export async function getLatestEntities({ + entitiesESClient, +}: { + entitiesESClient: EntitiesESClient; +}) { + const response = ( + await entitiesESClient.searchLatest('get_latest_entities', { + body: { + size: MAX_NUMBER_OF_ENTITIES, + }, + }) + ).hits.hits.map((hit) => hit._source); + + return response; +} diff --git a/x-pack/plugins/observability_solution/inventory/server/routes/entities/route.ts b/x-pack/plugins/observability_solution/inventory/server/routes/entities/route.ts index 0622ed32ac9dc..093e5ff399ed1 100644 --- a/x-pack/plugins/observability_solution/inventory/server/routes/entities/route.ts +++ b/x-pack/plugins/observability_solution/inventory/server/routes/entities/route.ts @@ -4,31 +4,28 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { i18n } from '@kbn/i18n'; -import type { EntityTypeDefinition } from '../../../common/entities'; import { createInventoryServerRoute } from '../create_inventory_server_route'; +import { createEntitiesESClient } from '../../lib/create_es_client/create_entities_es_client'; +import { getLatestEntities } from './get_latest_entities'; -export const listEntityTypesRoute = createInventoryServerRoute({ - endpoint: 'GET /internal/inventory/entity_types', +export const listLatestEntitiesRoute = createInventoryServerRoute({ + endpoint: 'GET /internal/inventory/entities', options: { tags: ['access:inventory'], }, - handler: async ({ plugins, request }): Promise<{ definitions: EntityTypeDefinition[] }> => { - return { - definitions: [ - { - label: i18n.translate('xpack.inventory.entityTypeLabels.datasets', { - defaultMessage: 'Datasets', - }), - icon: 'pipeNoBreaks', - type: 'dataset', - count: 0, - }, - ], - }; + handler: async ({ plugins, request, context }) => { + const coreContext = await context.core; + const entitiesESClient = createEntitiesESClient({ + esClient: coreContext.elasticsearch.client.asCurrentUser, + request, + }); + + const latestEntities = await getLatestEntities({ entitiesESClient }); + + return { entities: latestEntities }; }, }); export const entitiesRoutes = { - ...listEntityTypesRoute, + ...listLatestEntitiesRoute, }; diff --git a/x-pack/plugins/observability_solution/inventory/server/utils/with_apm_span.ts b/x-pack/plugins/observability_solution/inventory/server/utils/with_apm_span.ts new file mode 100644 index 0000000000000..b9e79df6cb0d7 --- /dev/null +++ b/x-pack/plugins/observability_solution/inventory/server/utils/with_apm_span.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +// export { withApmSpan } from '@kbn/apm-data-access-plugin/server/utils'; diff --git a/x-pack/plugins/observability_solution/inventory/tsconfig.json b/x-pack/plugins/observability_solution/inventory/tsconfig.json index 89fdd2e8fdf01..c0fc7c2692fde 100644 --- a/x-pack/plugins/observability_solution/inventory/tsconfig.json +++ b/x-pack/plugins/observability_solution/inventory/tsconfig.json @@ -24,7 +24,6 @@ "@kbn/server-route-repository", "@kbn/shared-ux-link-redirect-app", "@kbn/typed-react-router-config", - "@kbn/investigate-plugin", "@kbn/observability-utils", "@kbn/kibana-react-plugin", "@kbn/i18n", @@ -35,5 +34,7 @@ "@kbn/data-views-plugin", "@kbn/server-route-repository-client", "@kbn/react-kibana-context-render", + "@kbn/es-types", + "@kbn/entities-schema" ] } From e66450ea4c75365463698a16f23849bae7c9a0c0 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Tue, 17 Sep 2024 12:41:33 +0200 Subject: [PATCH 11/58] [CI] Run PR against chrome-beta (#192257) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary This PR's goal is to enable developers (or automation) to test against chrome-beta with a PR label - to adjust sensitive tests, or to foresee test breakages. ⚠️ Risk: a PR verified/built with the chrome beta setting, would pass the PR tests, but might not pass the on-merge suite in the end. Addresses: https://github.com/elastic/kibana-operations/issues/199 Depends on: https://github.com/elastic/ci-agent-images/pull/907 It highlights the errors visible only on the next versions: https://buildkite.com/elastic/kibana-pull-request/builds/233373 And it doesn't break with non-beta run: https://buildkite.com/elastic/kibana-pull-request/builds/233716 --- .../ci-stats/pick_test_group_run_order.ts | 32 +++++++++++++++++-- .buildkite/scripts/steps/test/ftr_configs.sh | 13 ++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts b/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts index 1f50bccbf8049..7379ab526321a 100644 --- a/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts +++ b/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts @@ -197,6 +197,32 @@ function getEnabledFtrConfigs(patterns?: string[]) { } } +/** + * Collects environment variables from labels on the PR + * TODO: extract this (and other functions from this big file) to a separate module + */ +function collectEnvFromLabels() { + const LABEL_MAPPING: Record> = { + 'ci:use-chrome-beta': { + USE_CHROME_BETA: 'true', + }, + }; + + const envFromlabels: Record = {}; + if (!process.env.GITHUB_PR_LABELS) { + return envFromlabels; + } else { + const labels = process.env.GITHUB_PR_LABELS.split(','); + labels.forEach((label) => { + const env = LABEL_MAPPING[label]; + if (env) { + Object.assign(envFromlabels, env); + } + }); + return envFromlabels; + } +} + export async function pickTestGroupRunOrder() { const bk = new BuildkiteClient(); const ciStats = new CiStatsClient(); @@ -273,9 +299,10 @@ export async function pickTestGroupRunOrder() { .filter(Boolean) : ['build']; - const FTR_EXTRA_ARGS: Record = process.env.FTR_EXTRA_ARGS + const ftrExtraArgs: Record = process.env.FTR_EXTRA_ARGS ? { FTR_EXTRA_ARGS: process.env.FTR_EXTRA_ARGS } : {}; + const envFromlabels: Record = collectEnvFromLabels(); const { defaultQueue, ftrConfigsByQueue } = getEnabledFtrConfigs(FTR_CONFIG_PATTERNS); @@ -514,7 +541,8 @@ export async function pickTestGroupRunOrder() { agents: expandAgentQueue(queue), env: { FTR_CONFIG_GROUP_KEY: key, - ...FTR_EXTRA_ARGS, + ...ftrExtraArgs, + ...envFromlabels, }, retry: { automatic: [ diff --git a/.buildkite/scripts/steps/test/ftr_configs.sh b/.buildkite/scripts/steps/test/ftr_configs.sh index e9500a2244707..67fbfd9c95b20 100755 --- a/.buildkite/scripts/steps/test/ftr_configs.sh +++ b/.buildkite/scripts/steps/test/ftr_configs.sh @@ -57,6 +57,19 @@ while read -r config; do start=$(date +%s) + if [[ "${USE_CHROME_BETA:-}" =~ ^(1|true)$ ]]; then + echo "USE_CHROME_BETA was set - using google-chrome-beta" + export TEST_BROWSER_BINARY_PATH="$(which google-chrome-beta)" + + # download the beta version of chromedriver + export CHROMEDRIVER_VERSION=$(curl https://googlechromelabs.github.io/chrome-for-testing/last-known-good-versions.json -s | jq -r '.channels.Beta.version') + export DETECT_CHROMEDRIVER_VERSION=false + node node_modules/chromedriver/install.js --chromedriver-force-download + + # set annotation on the build + buildkite-agent annotate --style info --context chrome-beta "This build uses Google Chrome Beta @ ${CHROMEDRIVER_VERSION}" + fi + # prevent non-zero exit code from breaking the loop set +e; node ./scripts/functional_tests \ From a87e7e8e6393f8ca5e3f9772d6c8b458229984ef Mon Sep 17 00:00:00 2001 From: Kerry Gallagher Date: Tue, 17 Sep 2024 12:20:33 +0100 Subject: [PATCH 12/58] [Logs] Use central log sources setting for logs context resolution in Discover (#192605) ## Summary Closes https://github.com/elastic/logs-dev/issues/171. Most of the noise in the PR is from making methods async and amending test mocks, the core logic changes are in `createLogsContextService`. --- .../kbn-discover-utils/src/__mocks__/index.ts | 1 + .../src/__mocks__/logs_context_service.ts | 14 ++++++++ .../data_types/logs/logs_context_service.ts | 36 ++++++++++++++----- packages/kbn-discover-utils/tsconfig.json | 3 +- src/plugins/discover/kibana.jsonc | 3 +- .../context_awareness/__mocks__/index.tsx | 10 ++++-- .../profile_provider_services.ts | 15 ++++---- .../register_profile_providers.test.ts | 7 ++-- .../register_profile_providers.ts | 9 +++-- src/plugins/discover/public/plugin.tsx | 13 +++---- src/plugins/discover/public/types.ts | 2 ++ src/plugins/discover/tsconfig.json | 3 +- .../logs_data_access/common/constants.ts | 2 +- 13 files changed, 87 insertions(+), 31 deletions(-) create mode 100644 packages/kbn-discover-utils/src/__mocks__/logs_context_service.ts diff --git a/packages/kbn-discover-utils/src/__mocks__/index.ts b/packages/kbn-discover-utils/src/__mocks__/index.ts index 148c9a6323140..9aa3005478051 100644 --- a/packages/kbn-discover-utils/src/__mocks__/index.ts +++ b/packages/kbn-discover-utils/src/__mocks__/index.ts @@ -10,3 +10,4 @@ export * from './data_view'; export * from './es_hits'; export * from './additional_field_groups'; +export * from './logs_context_service'; diff --git a/packages/kbn-discover-utils/src/__mocks__/logs_context_service.ts b/packages/kbn-discover-utils/src/__mocks__/logs_context_service.ts new file mode 100644 index 0000000000000..86655b2275b64 --- /dev/null +++ b/packages/kbn-discover-utils/src/__mocks__/logs_context_service.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { DEFAULT_ALLOWED_LOGS_BASE_PATTERNS_REGEXP, getLogsContextService } from '../data_types'; + +export const createLogsContextServiceMock = () => { + return getLogsContextService([DEFAULT_ALLOWED_LOGS_BASE_PATTERNS_REGEXP]); +}; diff --git a/packages/kbn-discover-utils/src/data_types/logs/logs_context_service.ts b/packages/kbn-discover-utils/src/data_types/logs/logs_context_service.ts index da858ce310e4e..7af3a723e7b14 100644 --- a/packages/kbn-discover-utils/src/data_types/logs/logs_context_service.ts +++ b/packages/kbn-discover-utils/src/data_types/logs/logs_context_service.ts @@ -8,15 +8,14 @@ */ import { createRegExpPatternFrom, testPatternAgainstAllowedList } from '@kbn/data-view-utils'; +import type { LogsDataAccessPluginStart } from '@kbn/logs-data-access-plugin/public'; export interface LogsContextService { isLogsIndexPattern(indexPattern: unknown): boolean; } -// eslint-disable-next-line @typescript-eslint/no-empty-interface export interface LogsContextServiceDeps { - // We will probably soon add uiSettings as a dependency - // to consume user configured indices + logsDataAccessPlugin?: LogsDataAccessPluginStart; } export const DEFAULT_ALLOWED_LOGS_BASE_PATTERNS = [ @@ -28,15 +27,36 @@ export const DEFAULT_ALLOWED_LOGS_BASE_PATTERNS = [ 'winlogbeat', ]; -export const createLogsContextService = (_deps: LogsContextServiceDeps = {}) => { - // This is initially an hard-coded set of well-known base patterns, - // we can extend this allowed list with any setting coming from uiSettings - const ALLOWED_LOGS_DATA_SOURCES = [createRegExpPatternFrom(DEFAULT_ALLOWED_LOGS_BASE_PATTERNS)]; +export const DEFAULT_ALLOWED_LOGS_BASE_PATTERNS_REGEXP = createRegExpPatternFrom( + DEFAULT_ALLOWED_LOGS_BASE_PATTERNS +); +export const createLogsContextService = async ({ + logsDataAccessPlugin, +}: LogsContextServiceDeps) => { + let logSources: string[] | undefined; + + if (logsDataAccessPlugin) { + const logSourcesService = logsDataAccessPlugin.services.logSourcesService; + logSources = (await logSourcesService.getLogSources()) + .map((logSource) => logSource.indexPattern) + .join(',') // TODO: Will be replaced by helper in: https://github.com/elastic/kibana/pull/192003 + .split(','); + } + + const ALLOWED_LOGS_DATA_SOURCES = [ + DEFAULT_ALLOWED_LOGS_BASE_PATTERNS_REGEXP, + ...(logSources ? logSources : []), + ]; + + return getLogsContextService(ALLOWED_LOGS_DATA_SOURCES); +}; + +export const getLogsContextService = (allowedDataSources: Array) => { const isLogsIndexPattern = (indexPattern: unknown) => { return ( typeof indexPattern === 'string' && - testPatternAgainstAllowedList(ALLOWED_LOGS_DATA_SOURCES)(indexPattern) + testPatternAgainstAllowedList(allowedDataSources)(indexPattern) ); }; diff --git a/packages/kbn-discover-utils/tsconfig.json b/packages/kbn-discover-utils/tsconfig.json index ef9ccaa7ab9b7..724051e5863c4 100644 --- a/packages/kbn-discover-utils/tsconfig.json +++ b/packages/kbn-discover-utils/tsconfig.json @@ -26,6 +26,7 @@ "@kbn/i18n", "@kbn/core-ui-settings-browser", "@kbn/ui-theme", - "@kbn/expressions-plugin" + "@kbn/expressions-plugin", + "@kbn/logs-data-access-plugin" ] } diff --git a/src/plugins/discover/kibana.jsonc b/src/plugins/discover/kibana.jsonc index e2e836ec528ee..1f5e25229df02 100644 --- a/src/plugins/discover/kibana.jsonc +++ b/src/plugins/discover/kibana.jsonc @@ -41,7 +41,8 @@ "globalSearch", "observabilityAIAssistant", "aiops", - "fieldsMetadata" + "fieldsMetadata", + "logsDataAccess" ], "requiredBundles": ["kibanaUtils", "kibanaReact", "unifiedSearch", "savedObjects"], "extraPublicDirs": ["common"] diff --git a/src/plugins/discover/public/context_awareness/__mocks__/index.tsx b/src/plugins/discover/public/context_awareness/__mocks__/index.tsx index 9c3c3d668b889..cd8ab77875afb 100644 --- a/src/plugins/discover/public/context_awareness/__mocks__/index.tsx +++ b/src/plugins/discover/public/context_awareness/__mocks__/index.tsx @@ -21,8 +21,8 @@ import { RootProfileService, SolutionType, } from '../profiles'; -import { createProfileProviderServices } from '../profile_providers/profile_provider_services'; import { ProfilesManager } from '../profiles_manager'; +import { createLogsContextServiceMock } from '@kbn/discover-utils/src/__mocks__'; export const createContextAwarenessMocks = ({ shouldRegisterProviders = true, @@ -156,7 +156,7 @@ export const createContextAwarenessMocks = ({ documentProfileServiceMock ); - const profileProviderServices = createProfileProviderServices(); + const profileProviderServices = createProfileProviderServicesMock(); return { rootProfileProviderMock, @@ -171,3 +171,9 @@ export const createContextAwarenessMocks = ({ profileProviderServices, }; }; + +const createProfileProviderServicesMock = () => { + return { + logsContextService: createLogsContextServiceMock(), + }; +}; diff --git a/src/plugins/discover/public/context_awareness/profile_providers/profile_provider_services.ts b/src/plugins/discover/public/context_awareness/profile_providers/profile_provider_services.ts index 7abca8d6d8520..a757f24308173 100644 --- a/src/plugins/discover/public/context_awareness/profile_providers/profile_provider_services.ts +++ b/src/plugins/discover/public/context_awareness/profile_providers/profile_provider_services.ts @@ -8,14 +8,13 @@ */ import { createLogsContextService, LogsContextService } from '@kbn/discover-utils'; +import type { LogsDataAccessPluginStart } from '@kbn/logs-data-access-plugin/public'; /** * Dependencies required by profile provider implementations */ -// eslint-disable-next-line @typescript-eslint/no-empty-interface export interface ProfileProviderDeps { - // We will probably soon add uiSettings as a dependency - // to consume user configured indices + logsDataAccessPlugin?: LogsDataAccessPluginStart; } /** @@ -33,10 +32,12 @@ export interface ProfileProviderServices { * @param _deps Profile provider dependencies * @returns Profile provider services */ -export const createProfileProviderServices = ( - _deps: ProfileProviderDeps = {} -): ProfileProviderServices => { +export const createProfileProviderServices = async ( + deps: ProfileProviderDeps = {} +): Promise => { return { - logsContextService: createLogsContextService(), + logsContextService: await createLogsContextService({ + logsDataAccessPlugin: deps.logsDataAccessPlugin, + }), }; }; diff --git a/src/plugins/discover/public/context_awareness/profile_providers/register_profile_providers.test.ts b/src/plugins/discover/public/context_awareness/profile_providers/register_profile_providers.test.ts index 6b0c22a3d0d08..1269441df2f21 100644 --- a/src/plugins/discover/public/context_awareness/profile_providers/register_profile_providers.test.ts +++ b/src/plugins/discover/public/context_awareness/profile_providers/register_profile_providers.test.ts @@ -8,6 +8,7 @@ */ import { createEsqlDataSource } from '../../../common/data_sources'; +import { DiscoverStartPlugins } from '../../types'; import { createContextAwarenessMocks } from '../__mocks__'; import { createExampleRootProfileProvider } from './example/example_root_pofile'; import { createExampleDataSourceProfileProvider } from './example/example_data_source_profile/profile'; @@ -73,7 +74,8 @@ describe('registerProfileProviders', () => { createContextAwarenessMocks({ shouldRegisterProviders: false, }); - registerProfileProviders({ + await registerProfileProviders({ + plugins: {} as DiscoverStartPlugins, rootProfileService: rootProfileServiceMock, dataSourceProfileService: dataSourceProfileServiceMock, documentProfileService: documentProfileServiceMock, @@ -108,7 +110,8 @@ describe('registerProfileProviders', () => { createContextAwarenessMocks({ shouldRegisterProviders: false, }); - registerProfileProviders({ + await registerProfileProviders({ + plugins: {} as DiscoverStartPlugins, rootProfileService: rootProfileServiceMock, dataSourceProfileService: dataSourceProfileServiceMock, documentProfileService: documentProfileServiceMock, diff --git a/src/plugins/discover/public/context_awareness/profile_providers/register_profile_providers.ts b/src/plugins/discover/public/context_awareness/profile_providers/register_profile_providers.ts index 9cd65320ac140..3bd7ee9926f2d 100644 --- a/src/plugins/discover/public/context_awareness/profile_providers/register_profile_providers.ts +++ b/src/plugins/discover/public/context_awareness/profile_providers/register_profile_providers.ts @@ -23,17 +23,20 @@ import { createProfileProviderServices, ProfileProviderServices, } from './profile_provider_services'; +import type { DiscoverStartPlugins } from '../../types'; /** * Register profile providers for root, data source, and document contexts to the profile profile services * @param options Register profile provider options */ -export const registerProfileProviders = ({ +export const registerProfileProviders = async ({ + plugins, rootProfileService, dataSourceProfileService, documentProfileService, enabledExperimentalProfileIds, }: { + plugins: DiscoverStartPlugins; /** * Root profile service */ @@ -51,7 +54,9 @@ export const registerProfileProviders = ({ */ enabledExperimentalProfileIds: string[]; }) => { - const providerServices = createProfileProviderServices(); + const providerServices = await createProfileProviderServices({ + logsDataAccessPlugin: plugins.logsDataAccess, + }); const rootProfileProviders = createRootProfileProviders(providerServices); const dataSourceProfileProviders = createDataSourceProfileProviders(providerServices); const documentProfileProviders = createDocumentProfileProviders(providerServices); diff --git a/src/plugins/discover/public/plugin.tsx b/src/plugins/discover/public/plugin.tsx index 5139e2342c988..a4e94205be7f4 100644 --- a/src/plugins/discover/public/plugin.tsx +++ b/src/plugins/discover/public/plugin.tsx @@ -183,7 +183,7 @@ export class DiscoverPlugin history: this.historyService.getHistory(), scopedHistory: this.scopedHistory, urlTracker: this.urlTracker!, - profilesManager: await this.createProfilesManager(), + profilesManager: await this.createProfilesManager({ plugins: discoverStartPlugins }), setHeaderActionMenu: params.setHeaderActionMenu, }); @@ -302,14 +302,15 @@ export class DiscoverPlugin } } - private createProfileServices = once(async () => { + private createProfileServices = once(async ({ plugins }: { plugins: DiscoverStartPlugins }) => { const { registerProfileProviders } = await import('./context_awareness/profile_providers'); const rootProfileService = new RootProfileService(); const dataSourceProfileService = new DataSourceProfileService(); const documentProfileService = new DocumentProfileService(); const enabledExperimentalProfileIds = this.experimentalFeatures.enabledProfiles ?? []; - registerProfileProviders({ + await registerProfileProviders({ + plugins, rootProfileService, dataSourceProfileService, documentProfileService, @@ -319,9 +320,9 @@ export class DiscoverPlugin return { rootProfileService, dataSourceProfileService, documentProfileService }; }); - private async createProfilesManager() { + private async createProfilesManager({ plugins }: { plugins: DiscoverStartPlugins }) { const { rootProfileService, dataSourceProfileService, documentProfileService } = - await this.createProfileServices(); + await this.createProfileServices({ plugins }); return new ProfilesManager( rootProfileService, @@ -367,7 +368,7 @@ export class DiscoverPlugin const getDiscoverServicesInternal = async () => { const [coreStart, deps] = await core.getStartServices(); - const profilesManager = await this.createProfilesManager(); + const profilesManager = await this.createProfilesManager({ plugins: deps }); return this.getDiscoverServices(coreStart, deps, profilesManager); }; diff --git a/src/plugins/discover/public/types.ts b/src/plugins/discover/public/types.ts index 2b6f70f74529a..916833bdce1d5 100644 --- a/src/plugins/discover/public/types.ts +++ b/src/plugins/discover/public/types.ts @@ -41,6 +41,7 @@ import type { import type { AiopsPluginStart } from '@kbn/aiops-plugin/public'; import type { DataVisualizerPluginStart } from '@kbn/data-visualizer-plugin/public'; import type { FieldsMetadataPublicStart } from '@kbn/fields-metadata-plugin/public'; +import type { LogsDataAccessPluginStart } from '@kbn/logs-data-access-plugin/public'; import { DiscoverAppLocator } from '../common'; import { DiscoverCustomizationContext } from './customizations'; import { type DiscoverContainerProps } from './components/discover_container'; @@ -170,4 +171,5 @@ export interface DiscoverStartPlugins { urlForwarding: UrlForwardingStart; usageCollection?: UsageCollectionSetup; fieldsMetadata: FieldsMetadataPublicStart; + logsDataAccess?: LogsDataAccessPluginStart; } diff --git a/src/plugins/discover/tsconfig.json b/src/plugins/discover/tsconfig.json index a363981a5d731..51e797b179952 100644 --- a/src/plugins/discover/tsconfig.json +++ b/src/plugins/discover/tsconfig.json @@ -95,7 +95,8 @@ "@kbn/presentation-containers", "@kbn/observability-ai-assistant-plugin", "@kbn/fields-metadata-plugin", - "@kbn/security-solution-common" + "@kbn/security-solution-common", + "@kbn/logs-data-access-plugin" ], "exclude": [ "target/**/*" diff --git a/x-pack/plugins/observability_solution/logs_data_access/common/constants.ts b/x-pack/plugins/observability_solution/logs_data_access/common/constants.ts index c0caaa846f56e..ffe5a83f245e8 100644 --- a/x-pack/plugins/observability_solution/logs_data_access/common/constants.ts +++ b/x-pack/plugins/observability_solution/logs_data_access/common/constants.ts @@ -5,4 +5,4 @@ * 2.0. */ -export const DEFAULT_LOG_SOURCES = ['logs-*-*,logs-*,filebeat-*,kibana_sample_data_logs*']; +export const DEFAULT_LOG_SOURCES = ['logs-*-*', 'logs-*', 'filebeat-*', 'kibana_sample_data_logs*']; From 5c5897966ab0d6caaba64d943d91d0485dde16f2 Mon Sep 17 00:00:00 2001 From: natasha-moore-elastic <137783811+natasha-moore-elastic@users.noreply.github.com> Date: Tue, 17 Sep 2024 12:24:07 +0100 Subject: [PATCH 13/58] Improves Lists API docs content (#192504) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Resolves https://github.com/elastic/security-docs-internal/issues/32 by improving the Lists API docs content. Adds missing and improves existing operation summaries and operation descriptions to adhere to our [OAS standards](https://elasticco.atlassian.net/wiki/spaces/DOC/pages/450494532/API+reference+docs). Note: Couldn’t add description for the GET /api/lists/privileges operation, since it's not documented in [ESS API docs](https://www.elastic.co/guide/en/security/8.15/security-apis.html). --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../api/create_list/create_list.schema.yaml | 3 +- .../create_list_index.schema.yaml | 3 +- .../create_list_item.schema.yaml | 8 +- .../api/delete_list/delete_list.schema.yaml | 6 +- .../delete_list_index.schema.yaml | 3 +- .../delete_list_item.schema.yaml | 3 +- .../export_list_items.schema.yaml | 4 +- .../find_list_items.schema.yaml | 3 +- .../api/find_lists/find_lists.schema.yaml | 3 +- .../import_list_items.schema.yaml | 4 +- .../api/patch_list/patch_list.schema.yaml | 3 +- .../patch_list_item.schema.yaml | 3 +- .../api/quickstart_client.gen.ts | 63 +++++++++++++- .../api/read_list/read_list.schema.yaml | 3 +- .../read_list_index.schema.yaml | 3 +- .../read_list_item/read_list_item.schema.yaml | 3 +- .../read_list_privileges.schema.yaml | 2 +- .../api/update_list/update_list.schema.yaml | 6 +- .../update_list_item.schema.yaml | 6 +- ...n_lists_api_2023_10_31.bundled.schema.yaml | 83 ++++++++++++++----- ...n_lists_api_2023_10_31.bundled.schema.yaml | 83 ++++++++++++++----- .../security_solution_lists_api.gen.ts | 63 +++++++++++++- 22 files changed, 295 insertions(+), 66 deletions(-) diff --git a/packages/kbn-securitysolution-lists-common/api/create_list/create_list.schema.yaml b/packages/kbn-securitysolution-lists-common/api/create_list/create_list.schema.yaml index bae6565b0f1e0..191e973beba61 100644 --- a/packages/kbn-securitysolution-lists-common/api/create_list/create_list.schema.yaml +++ b/packages/kbn-securitysolution-lists-common/api/create_list/create_list.schema.yaml @@ -8,7 +8,8 @@ paths: x-labels: [serverless, ess] operationId: CreateList x-codegen-enabled: true - summary: Creates a list + summary: Create a list + description: Create a new list. requestBody: description: List's properties required: true diff --git a/packages/kbn-securitysolution-lists-common/api/create_list_index/create_list_index.schema.yaml b/packages/kbn-securitysolution-lists-common/api/create_list_index/create_list_index.schema.yaml index dcb1baa3ee3d8..c775a9c7d873f 100644 --- a/packages/kbn-securitysolution-lists-common/api/create_list_index/create_list_index.schema.yaml +++ b/packages/kbn-securitysolution-lists-common/api/create_list_index/create_list_index.schema.yaml @@ -8,7 +8,8 @@ paths: x-labels: [serverless, ess] operationId: CreateListIndex x-codegen-enabled: true - summary: Creates necessary list data streams + summary: Create list data streams + description: Create `.lists` and `.items` data streams in the relevant space. responses: 200: description: Successful response diff --git a/packages/kbn-securitysolution-lists-common/api/create_list_item/create_list_item.schema.yaml b/packages/kbn-securitysolution-lists-common/api/create_list_item/create_list_item.schema.yaml index 10fef88e7a042..01121d0143925 100644 --- a/packages/kbn-securitysolution-lists-common/api/create_list_item/create_list_item.schema.yaml +++ b/packages/kbn-securitysolution-lists-common/api/create_list_item/create_list_item.schema.yaml @@ -8,7 +8,13 @@ paths: x-labels: [serverless, ess] operationId: CreateListItem x-codegen-enabled: true - summary: Creates a list item + summary: Create a list item + description: | + Create a list item and associate it with the specified list. + + All list items in the same list must be the same type. For example, each list item in an `ip` list must define a specific IP address. + > info + > Before creating a list item, you must create a list. requestBody: description: List item's properties required: true diff --git a/packages/kbn-securitysolution-lists-common/api/delete_list/delete_list.schema.yaml b/packages/kbn-securitysolution-lists-common/api/delete_list/delete_list.schema.yaml index 4a5974003918e..7098753636379 100644 --- a/packages/kbn-securitysolution-lists-common/api/delete_list/delete_list.schema.yaml +++ b/packages/kbn-securitysolution-lists-common/api/delete_list/delete_list.schema.yaml @@ -8,7 +8,11 @@ paths: x-labels: [serverless, ess] operationId: DeleteList x-codegen-enabled: true - summary: Deletes a list + summary: Delete a list + description: | + Delete a list using the list ID. + > info + > When you delete a list, all of its list items are also deleted. parameters: - name: id in: query diff --git a/packages/kbn-securitysolution-lists-common/api/delete_list_index/delete_list_index.schema.yaml b/packages/kbn-securitysolution-lists-common/api/delete_list_index/delete_list_index.schema.yaml index 34dcf91f548d0..4f4b0f00e8817 100644 --- a/packages/kbn-securitysolution-lists-common/api/delete_list_index/delete_list_index.schema.yaml +++ b/packages/kbn-securitysolution-lists-common/api/delete_list_index/delete_list_index.schema.yaml @@ -8,7 +8,8 @@ paths: x-labels: [serverless, ess] operationId: DeleteListIndex x-codegen-enabled: true - summary: Deletes list data streams + summary: Delete list data streams + description: Delete the `.lists` and `.items` data streams. responses: 200: description: Successful response diff --git a/packages/kbn-securitysolution-lists-common/api/delete_list_item/delete_list_item.schema.yaml b/packages/kbn-securitysolution-lists-common/api/delete_list_item/delete_list_item.schema.yaml index 413c85b55dd3b..28913259387dd 100644 --- a/packages/kbn-securitysolution-lists-common/api/delete_list_item/delete_list_item.schema.yaml +++ b/packages/kbn-securitysolution-lists-common/api/delete_list_item/delete_list_item.schema.yaml @@ -8,7 +8,8 @@ paths: x-labels: [serverless, ess] operationId: DeleteListItem x-codegen-enabled: true - summary: Deletes a list item + summary: Delete a list item + description: Delete a list item using its `id`, or its `list_id` and `value` fields. parameters: - name: id in: query diff --git a/packages/kbn-securitysolution-lists-common/api/export_list_items/export_list_items.schema.yaml b/packages/kbn-securitysolution-lists-common/api/export_list_items/export_list_items.schema.yaml index 06eda41d042fb..8d185a23b64c9 100644 --- a/packages/kbn-securitysolution-lists-common/api/export_list_items/export_list_items.schema.yaml +++ b/packages/kbn-securitysolution-lists-common/api/export_list_items/export_list_items.schema.yaml @@ -8,8 +8,8 @@ paths: x-labels: [serverless, ess] operationId: ExportListItems x-codegen-enabled: true - summary: Exports list items - description: Exports list item values from the specified list + summary: Export list items + description: Export list item values from the specified list. parameters: - name: list_id in: query diff --git a/packages/kbn-securitysolution-lists-common/api/find_list_items/find_list_items.schema.yaml b/packages/kbn-securitysolution-lists-common/api/find_list_items/find_list_items.schema.yaml index b08e7b4719374..21cf4ffd61841 100644 --- a/packages/kbn-securitysolution-lists-common/api/find_list_items/find_list_items.schema.yaml +++ b/packages/kbn-securitysolution-lists-common/api/find_list_items/find_list_items.schema.yaml @@ -8,7 +8,8 @@ paths: x-labels: [serverless, ess] operationId: FindListItems x-codegen-enabled: true - summary: Finds list items + summary: Get list items + description: Get all list items in the specified list. parameters: - name: list_id in: query diff --git a/packages/kbn-securitysolution-lists-common/api/find_lists/find_lists.schema.yaml b/packages/kbn-securitysolution-lists-common/api/find_lists/find_lists.schema.yaml index 071dba08254ec..3bb55decacff6 100644 --- a/packages/kbn-securitysolution-lists-common/api/find_lists/find_lists.schema.yaml +++ b/packages/kbn-securitysolution-lists-common/api/find_lists/find_lists.schema.yaml @@ -8,7 +8,8 @@ paths: x-labels: [serverless, ess] operationId: FindLists x-codegen-enabled: true - summary: Finds lists + summary: Get lists + description: Get a paginated subset of lists. By default, the first page is returned, with 20 results per page. parameters: - name: page in: query diff --git a/packages/kbn-securitysolution-lists-common/api/import_list_items/import_list_items.schema.yaml b/packages/kbn-securitysolution-lists-common/api/import_list_items/import_list_items.schema.yaml index 895e222c05207..520213e949c1d 100644 --- a/packages/kbn-securitysolution-lists-common/api/import_list_items/import_list_items.schema.yaml +++ b/packages/kbn-securitysolution-lists-common/api/import_list_items/import_list_items.schema.yaml @@ -8,9 +8,9 @@ paths: x-labels: [serverless, ess] operationId: ImportListItems x-codegen-enabled: true - summary: Imports list items + summary: Import list items description: | - Imports a list of items from a `.txt` or `.csv` file. The maximum file size is 9 million bytes. + Import list items from a TXT or CSV file. The maximum file size is 9 million bytes. You can import items to a new or existing list. requestBody: diff --git a/packages/kbn-securitysolution-lists-common/api/patch_list/patch_list.schema.yaml b/packages/kbn-securitysolution-lists-common/api/patch_list/patch_list.schema.yaml index 1ca568acb2bbc..b98b34e6347eb 100644 --- a/packages/kbn-securitysolution-lists-common/api/patch_list/patch_list.schema.yaml +++ b/packages/kbn-securitysolution-lists-common/api/patch_list/patch_list.schema.yaml @@ -8,7 +8,8 @@ paths: x-labels: [serverless, ess] operationId: PatchList x-codegen-enabled: true - summary: Patches a list + summary: Patch a list + description: Update specific fields of an existing list using the list ID. requestBody: description: List's properties required: true diff --git a/packages/kbn-securitysolution-lists-common/api/patch_list_item/patch_list_item.schema.yaml b/packages/kbn-securitysolution-lists-common/api/patch_list_item/patch_list_item.schema.yaml index a17982db14452..f79efc4691dde 100644 --- a/packages/kbn-securitysolution-lists-common/api/patch_list_item/patch_list_item.schema.yaml +++ b/packages/kbn-securitysolution-lists-common/api/patch_list_item/patch_list_item.schema.yaml @@ -8,7 +8,8 @@ paths: x-labels: [serverless, ess] operationId: PatchListItem x-codegen-enabled: true - summary: Patches a list item + summary: Patch a list item + description: Update specific fields of an existing list item using the list item ID. requestBody: description: List item's properties required: true diff --git a/packages/kbn-securitysolution-lists-common/api/quickstart_client.gen.ts b/packages/kbn-securitysolution-lists-common/api/quickstart_client.gen.ts index a6e314cd5d717..7bf343d935f2c 100644 --- a/packages/kbn-securitysolution-lists-common/api/quickstart_client.gen.ts +++ b/packages/kbn-securitysolution-lists-common/api/quickstart_client.gen.ts @@ -77,6 +77,9 @@ export class Client { this.kbnClient = options.kbnClient; this.log = options.log; } + /** + * Create a new list. + */ async createList(props: CreateListProps) { this.log.info(`${new Date().toISOString()} Calling API CreateList`); return this.kbnClient @@ -90,6 +93,9 @@ export class Client { }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Create `.lists` and `.items` data streams in the relevant space. + */ async createListIndex() { this.log.info(`${new Date().toISOString()} Calling API CreateListIndex`); return this.kbnClient @@ -102,6 +108,14 @@ export class Client { }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Create a list item and associate it with the specified list. + +All list items in the same list must be the same type. For example, each list item in an `ip` list must define a specific IP address. +> info +> Before creating a list item, you must create a list. + + */ async createListItem(props: CreateListItemProps) { this.log.info(`${new Date().toISOString()} Calling API CreateListItem`); return this.kbnClient @@ -115,6 +129,12 @@ export class Client { }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Delete a list using the list ID. +> info +> When you delete a list, all of its list items are also deleted. + + */ async deleteList(props: DeleteListProps) { this.log.info(`${new Date().toISOString()} Calling API DeleteList`); return this.kbnClient @@ -129,6 +149,9 @@ export class Client { }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Delete the `.lists` and `.items` data streams. + */ async deleteListIndex() { this.log.info(`${new Date().toISOString()} Calling API DeleteListIndex`); return this.kbnClient @@ -141,6 +164,9 @@ export class Client { }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Delete a list item using its `id`, or its `list_id` and `value` fields. + */ async deleteListItem(props: DeleteListItemProps) { this.log.info(`${new Date().toISOString()} Calling API DeleteListItem`); return this.kbnClient @@ -156,7 +182,7 @@ export class Client { .catch(catchAxiosErrorFormatAndThrow); } /** - * Exports list item values from the specified list + * Export list item values from the specified list. */ async exportListItems(props: ExportListItemsProps) { this.log.info(`${new Date().toISOString()} Calling API ExportListItems`); @@ -172,6 +198,9 @@ export class Client { }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Get all list items in the specified list. + */ async findListItems(props: FindListItemsProps) { this.log.info(`${new Date().toISOString()} Calling API FindListItems`); return this.kbnClient @@ -186,6 +215,9 @@ export class Client { }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Get a paginated subset of lists. By default, the first page is returned, with 20 results per page. + */ async findLists(props: FindListsProps) { this.log.info(`${new Date().toISOString()} Calling API FindLists`); return this.kbnClient @@ -201,7 +233,7 @@ export class Client { .catch(catchAxiosErrorFormatAndThrow); } /** - * Imports a list of items from a `.txt` or `.csv` file. The maximum file size is 9 million bytes. + * Import list items from a TXT or CSV file. The maximum file size is 9 million bytes. You can import items to a new or existing list. @@ -220,6 +252,9 @@ You can import items to a new or existing list. }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Update specific fields of an existing list using the list ID. + */ async patchList(props: PatchListProps) { this.log.info(`${new Date().toISOString()} Calling API PatchList`); return this.kbnClient @@ -233,6 +268,9 @@ You can import items to a new or existing list. }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Update specific fields of an existing list item using the list item ID. + */ async patchListItem(props: PatchListItemProps) { this.log.info(`${new Date().toISOString()} Calling API PatchListItem`); return this.kbnClient @@ -246,6 +284,9 @@ You can import items to a new or existing list. }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Get the details of a list using the list ID. + */ async readList(props: ReadListProps) { this.log.info(`${new Date().toISOString()} Calling API ReadList`); return this.kbnClient @@ -260,6 +301,9 @@ You can import items to a new or existing list. }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Verify that `.lists` and `.items` data streams exist. + */ async readListIndex() { this.log.info(`${new Date().toISOString()} Calling API ReadListIndex`); return this.kbnClient @@ -272,6 +316,9 @@ You can import items to a new or existing list. }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Get the details of a list item. + */ async readListItem(props: ReadListItemProps) { this.log.info(`${new Date().toISOString()} Calling API ReadListItem`); return this.kbnClient @@ -298,6 +345,12 @@ You can import items to a new or existing list. }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Update a list using the list ID. The original list is replaced, and all unspecified fields are deleted. +> info +> You cannot modify the `id` value. + + */ async updateList(props: UpdateListProps) { this.log.info(`${new Date().toISOString()} Calling API UpdateList`); return this.kbnClient @@ -311,6 +364,12 @@ You can import items to a new or existing list. }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Update a list item using the list item ID. The original list item is replaced, and all unspecified fields are deleted. +> info +> You cannot modify the `id` value. + + */ async updateListItem(props: UpdateListItemProps) { this.log.info(`${new Date().toISOString()} Calling API UpdateListItem`); return this.kbnClient diff --git a/packages/kbn-securitysolution-lists-common/api/read_list/read_list.schema.yaml b/packages/kbn-securitysolution-lists-common/api/read_list/read_list.schema.yaml index 770c6fa8a9e7d..d932e16f528a5 100644 --- a/packages/kbn-securitysolution-lists-common/api/read_list/read_list.schema.yaml +++ b/packages/kbn-securitysolution-lists-common/api/read_list/read_list.schema.yaml @@ -8,7 +8,8 @@ paths: x-labels: [serverless, ess] operationId: ReadList x-codegen-enabled: true - summary: Retrieves a list using its id field + summary: Get list details + description: Get the details of a list using the list ID. parameters: - name: id in: query diff --git a/packages/kbn-securitysolution-lists-common/api/read_list_index/read_list_index.schema.yaml b/packages/kbn-securitysolution-lists-common/api/read_list_index/read_list_index.schema.yaml index 3706563c008ce..b675264600157 100644 --- a/packages/kbn-securitysolution-lists-common/api/read_list_index/read_list_index.schema.yaml +++ b/packages/kbn-securitysolution-lists-common/api/read_list_index/read_list_index.schema.yaml @@ -8,7 +8,8 @@ paths: x-labels: [serverless, ess] operationId: ReadListIndex x-codegen-enabled: true - summary: Get list data stream existence status + summary: Get status of list data streams + description: Verify that `.lists` and `.items` data streams exist. responses: 200: description: Successful response diff --git a/packages/kbn-securitysolution-lists-common/api/read_list_item/read_list_item.schema.yaml b/packages/kbn-securitysolution-lists-common/api/read_list_item/read_list_item.schema.yaml index fe4a6046f012c..4d686f5452e0c 100644 --- a/packages/kbn-securitysolution-lists-common/api/read_list_item/read_list_item.schema.yaml +++ b/packages/kbn-securitysolution-lists-common/api/read_list_item/read_list_item.schema.yaml @@ -8,7 +8,8 @@ paths: x-labels: [serverless, ess] operationId: ReadListItem x-codegen-enabled: true - summary: Gets a list item + summary: Get a list item + description: Get the details of a list item. parameters: - name: id in: query diff --git a/packages/kbn-securitysolution-lists-common/api/read_list_privileges/read_list_privileges.schema.yaml b/packages/kbn-securitysolution-lists-common/api/read_list_privileges/read_list_privileges.schema.yaml index fd22321c9ed29..ec8604e80694e 100644 --- a/packages/kbn-securitysolution-lists-common/api/read_list_privileges/read_list_privileges.schema.yaml +++ b/packages/kbn-securitysolution-lists-common/api/read_list_privileges/read_list_privileges.schema.yaml @@ -8,7 +8,7 @@ paths: x-labels: [serverless, ess] operationId: ReadListPrivileges x-codegen-enabled: true - summary: Gets list privileges + summary: Get list privileges responses: 200: description: Successful response diff --git a/packages/kbn-securitysolution-lists-common/api/update_list/update_list.schema.yaml b/packages/kbn-securitysolution-lists-common/api/update_list/update_list.schema.yaml index b31bea393c91b..c41b52427b63d 100644 --- a/packages/kbn-securitysolution-lists-common/api/update_list/update_list.schema.yaml +++ b/packages/kbn-securitysolution-lists-common/api/update_list/update_list.schema.yaml @@ -8,7 +8,11 @@ paths: x-labels: [serverless, ess] operationId: UpdateList x-codegen-enabled: true - summary: Updates a list + summary: Update a list + description: | + Update a list using the list ID. The original list is replaced, and all unspecified fields are deleted. + > info + > You cannot modify the `id` value. requestBody: description: List's properties required: true diff --git a/packages/kbn-securitysolution-lists-common/api/update_list_item/update_list_item.schema.yaml b/packages/kbn-securitysolution-lists-common/api/update_list_item/update_list_item.schema.yaml index 95a4df349ff93..6b05e01f35aab 100644 --- a/packages/kbn-securitysolution-lists-common/api/update_list_item/update_list_item.schema.yaml +++ b/packages/kbn-securitysolution-lists-common/api/update_list_item/update_list_item.schema.yaml @@ -8,7 +8,11 @@ paths: x-labels: [serverless, ess] operationId: UpdateListItem x-codegen-enabled: true - summary: Updates a list item + summary: Update a list item + description: | + Update a list item using the list item ID. The original list item is replaced, and all unspecified fields are deleted. + > info + > You cannot modify the `id` value. requestBody: description: List item's properties required: true diff --git a/packages/kbn-securitysolution-lists-common/docs/openapi/ess/security_solution_lists_api_2023_10_31.bundled.schema.yaml b/packages/kbn-securitysolution-lists-common/docs/openapi/ess/security_solution_lists_api_2023_10_31.bundled.schema.yaml index 7fdb215489101..2db10e5afbcec 100644 --- a/packages/kbn-securitysolution-lists-common/docs/openapi/ess/security_solution_lists_api_2023_10_31.bundled.schema.yaml +++ b/packages/kbn-securitysolution-lists-common/docs/openapi/ess/security_solution_lists_api_2023_10_31.bundled.schema.yaml @@ -13,6 +13,10 @@ servers: paths: /api/lists: delete: + description: | + Delete a list using the list ID. + > info + > When you delete a list, all of its list items are also deleted. operationId: DeleteList parameters: - description: List's `id` value @@ -72,10 +76,11 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Deletes a list + summary: Delete a list tags: - Security Solution Lists API get: + description: Get the details of a list using the list ID. operationId: ReadList parameters: - description: List's `id` value @@ -123,10 +128,11 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Retrieves a list using its id field + summary: Get list details tags: - Security Solution Lists API patch: + description: Update specific fields of an existing list using the list ID. operationId: PatchList requestBody: content: @@ -190,10 +196,11 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Patches a list + summary: Patch a list tags: - Security Solution Lists API post: + description: Create a new list. operationId: CreateList requestBody: content: @@ -264,10 +271,17 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Creates a list + summary: Create a list tags: - Security Solution Lists API put: + description: > + Update a list using the list ID. The original list is replaced, and all + unspecified fields are deleted. + + > info + + > You cannot modify the `id` value. operationId: UpdateList requestBody: content: @@ -333,11 +347,14 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Updates a list + summary: Update a list tags: - Security Solution Lists API /api/lists/_find: get: + description: >- + Get a paginated subset of lists. By default, the first page is returned, + with 20 results per page. operationId: FindLists parameters: - description: The page number to return @@ -446,11 +463,12 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Finds lists + summary: Get lists tags: - Security Solution Lists API /api/lists/index: delete: + description: Delete the `.lists` and `.items` data streams. operationId: DeleteListIndex responses: '200': @@ -496,10 +514,11 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Deletes list data streams + summary: Delete list data streams tags: - Security Solution Lists API get: + description: Verify that `.lists` and `.items` data streams exist. operationId: ReadListIndex responses: '200': @@ -548,10 +567,11 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Get list data stream existence status + summary: Get status of list data streams tags: - Security Solution Lists API post: + description: Create `.lists` and `.items` data streams in the relevant space. operationId: CreateListIndex responses: '200': @@ -597,11 +617,12 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Creates necessary list data streams + summary: Create list data streams tags: - Security Solution Lists API /api/lists/items: delete: + description: 'Delete a list item using its `id`, or its `list_id` and `value` fields.' operationId: DeleteListItem parameters: - description: Required if `list_id` and `value` are not specified @@ -678,10 +699,11 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Deletes a list item + summary: Delete a list item tags: - Security Solution Lists API get: + description: Get the details of a list item. operationId: ReadListItem parameters: - description: Required if `list_id` and `value` are not specified @@ -745,10 +767,11 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Gets a list item + summary: Get a list item tags: - Security Solution Lists API patch: + description: Update specific fields of an existing list item using the list item ID. operationId: PatchListItem requestBody: content: @@ -816,10 +839,20 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Patches a list item + summary: Patch a list item tags: - Security Solution Lists API post: + description: > + Create a list item and associate it with the specified list. + + + All list items in the same list must be the same type. For example, each + list item in an `ip` list must define a specific IP address. + + > info + + > Before creating a list item, you must create a list. operationId: CreateListItem requestBody: content: @@ -888,10 +921,17 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Creates a list item + summary: Create a list item tags: - Security Solution Lists API put: + description: > + Update a list item using the list item ID. The original list item is + replaced, and all unspecified fields are deleted. + + > info + + > You cannot modify the `id` value. operationId: UpdateListItem requestBody: content: @@ -951,12 +991,12 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Updates a list item + summary: Update a list item tags: - Security Solution Lists API /api/lists/items/_export: post: - description: Exports list item values from the specified list + description: Export list item values from the specified list. operationId: ExportListItems parameters: - description: List's id to export @@ -1006,11 +1046,12 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Exports list items + summary: Export list items tags: - Security Solution Lists API /api/lists/items/_find: get: + description: Get all list items in the specified list. operationId: FindListItems parameters: - description: List's id @@ -1125,14 +1166,14 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Finds list items + summary: Get list items tags: - Security Solution Lists API /api/lists/items/_import: post: description: > - Imports a list of items from a `.txt` or `.csv` file. The maximum file - size is 9 million bytes. + Import list items from a TXT or CSV file. The maximum file size is 9 + million bytes. You can import items to a new or existing list. @@ -1232,7 +1273,7 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Imports list items + summary: Import list items tags: - Security Solution Lists API /api/lists/privileges: @@ -1282,7 +1323,7 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Gets list privileges + summary: Get list privileges tags: - Security Solution Lists API components: diff --git a/packages/kbn-securitysolution-lists-common/docs/openapi/serverless/security_solution_lists_api_2023_10_31.bundled.schema.yaml b/packages/kbn-securitysolution-lists-common/docs/openapi/serverless/security_solution_lists_api_2023_10_31.bundled.schema.yaml index c55ffe963a607..4f91b5112bfd0 100644 --- a/packages/kbn-securitysolution-lists-common/docs/openapi/serverless/security_solution_lists_api_2023_10_31.bundled.schema.yaml +++ b/packages/kbn-securitysolution-lists-common/docs/openapi/serverless/security_solution_lists_api_2023_10_31.bundled.schema.yaml @@ -13,6 +13,10 @@ servers: paths: /api/lists: delete: + description: | + Delete a list using the list ID. + > info + > When you delete a list, all of its list items are also deleted. operationId: DeleteList parameters: - description: List's `id` value @@ -72,10 +76,11 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Deletes a list + summary: Delete a list tags: - Security Solution Lists API get: + description: Get the details of a list using the list ID. operationId: ReadList parameters: - description: List's `id` value @@ -123,10 +128,11 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Retrieves a list using its id field + summary: Get list details tags: - Security Solution Lists API patch: + description: Update specific fields of an existing list using the list ID. operationId: PatchList requestBody: content: @@ -190,10 +196,11 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Patches a list + summary: Patch a list tags: - Security Solution Lists API post: + description: Create a new list. operationId: CreateList requestBody: content: @@ -264,10 +271,17 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Creates a list + summary: Create a list tags: - Security Solution Lists API put: + description: > + Update a list using the list ID. The original list is replaced, and all + unspecified fields are deleted. + + > info + + > You cannot modify the `id` value. operationId: UpdateList requestBody: content: @@ -333,11 +347,14 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Updates a list + summary: Update a list tags: - Security Solution Lists API /api/lists/_find: get: + description: >- + Get a paginated subset of lists. By default, the first page is returned, + with 20 results per page. operationId: FindLists parameters: - description: The page number to return @@ -446,11 +463,12 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Finds lists + summary: Get lists tags: - Security Solution Lists API /api/lists/index: delete: + description: Delete the `.lists` and `.items` data streams. operationId: DeleteListIndex responses: '200': @@ -496,10 +514,11 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Deletes list data streams + summary: Delete list data streams tags: - Security Solution Lists API get: + description: Verify that `.lists` and `.items` data streams exist. operationId: ReadListIndex responses: '200': @@ -548,10 +567,11 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Get list data stream existence status + summary: Get status of list data streams tags: - Security Solution Lists API post: + description: Create `.lists` and `.items` data streams in the relevant space. operationId: CreateListIndex responses: '200': @@ -597,11 +617,12 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Creates necessary list data streams + summary: Create list data streams tags: - Security Solution Lists API /api/lists/items: delete: + description: 'Delete a list item using its `id`, or its `list_id` and `value` fields.' operationId: DeleteListItem parameters: - description: Required if `list_id` and `value` are not specified @@ -678,10 +699,11 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Deletes a list item + summary: Delete a list item tags: - Security Solution Lists API get: + description: Get the details of a list item. operationId: ReadListItem parameters: - description: Required if `list_id` and `value` are not specified @@ -745,10 +767,11 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Gets a list item + summary: Get a list item tags: - Security Solution Lists API patch: + description: Update specific fields of an existing list item using the list item ID. operationId: PatchListItem requestBody: content: @@ -816,10 +839,20 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Patches a list item + summary: Patch a list item tags: - Security Solution Lists API post: + description: > + Create a list item and associate it with the specified list. + + + All list items in the same list must be the same type. For example, each + list item in an `ip` list must define a specific IP address. + + > info + + > Before creating a list item, you must create a list. operationId: CreateListItem requestBody: content: @@ -888,10 +921,17 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Creates a list item + summary: Create a list item tags: - Security Solution Lists API put: + description: > + Update a list item using the list item ID. The original list item is + replaced, and all unspecified fields are deleted. + + > info + + > You cannot modify the `id` value. operationId: UpdateListItem requestBody: content: @@ -951,12 +991,12 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Updates a list item + summary: Update a list item tags: - Security Solution Lists API /api/lists/items/_export: post: - description: Exports list item values from the specified list + description: Export list item values from the specified list. operationId: ExportListItems parameters: - description: List's id to export @@ -1006,11 +1046,12 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Exports list items + summary: Export list items tags: - Security Solution Lists API /api/lists/items/_find: get: + description: Get all list items in the specified list. operationId: FindListItems parameters: - description: List's id @@ -1125,14 +1166,14 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Finds list items + summary: Get list items tags: - Security Solution Lists API /api/lists/items/_import: post: description: > - Imports a list of items from a `.txt` or `.csv` file. The maximum file - size is 9 million bytes. + Import list items from a TXT or CSV file. The maximum file size is 9 + million bytes. You can import items to a new or existing list. @@ -1232,7 +1273,7 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Imports list items + summary: Import list items tags: - Security Solution Lists API /api/lists/privileges: @@ -1282,7 +1323,7 @@ paths: schema: $ref: '#/components/schemas/SiemErrorResponse' description: Internal server error response - summary: Gets list privileges + summary: Get list privileges tags: - Security Solution Lists API components: diff --git a/x-pack/test/api_integration/services/security_solution_lists_api.gen.ts b/x-pack/test/api_integration/services/security_solution_lists_api.gen.ts index 4ebd688bf296a..6ae8c4d1d4903 100644 --- a/x-pack/test/api_integration/services/security_solution_lists_api.gen.ts +++ b/x-pack/test/api_integration/services/security_solution_lists_api.gen.ts @@ -39,6 +39,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) const supertest = getService('supertest'); return { + /** + * Create a new list. + */ createList(props: CreateListProps) { return supertest .post('/api/lists') @@ -47,6 +50,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .send(props.body as object); }, + /** + * Create `.lists` and `.items` data streams in the relevant space. + */ createListIndex() { return supertest .post('/api/lists/index') @@ -54,6 +60,14 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(ELASTIC_HTTP_VERSION_HEADER, '2023-10-31') .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); }, + /** + * Create a list item and associate it with the specified list. + +All list items in the same list must be the same type. For example, each list item in an `ip` list must define a specific IP address. +> info +> Before creating a list item, you must create a list. + + */ createListItem(props: CreateListItemProps) { return supertest .post('/api/lists/items') @@ -62,6 +76,12 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .send(props.body as object); }, + /** + * Delete a list using the list ID. +> info +> When you delete a list, all of its list items are also deleted. + + */ deleteList(props: DeleteListProps) { return supertest .delete('/api/lists') @@ -70,6 +90,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .query(props.query); }, + /** + * Delete the `.lists` and `.items` data streams. + */ deleteListIndex() { return supertest .delete('/api/lists/index') @@ -77,6 +100,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(ELASTIC_HTTP_VERSION_HEADER, '2023-10-31') .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); }, + /** + * Delete a list item using its `id`, or its `list_id` and `value` fields. + */ deleteListItem(props: DeleteListItemProps) { return supertest .delete('/api/lists/items') @@ -86,7 +112,7 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .query(props.query); }, /** - * Exports list item values from the specified list + * Export list item values from the specified list. */ exportListItems(props: ExportListItemsProps) { return supertest @@ -96,6 +122,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .query(props.query); }, + /** + * Get all list items in the specified list. + */ findListItems(props: FindListItemsProps) { return supertest .get('/api/lists/items/_find') @@ -104,6 +133,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .query(props.query); }, + /** + * Get a paginated subset of lists. By default, the first page is returned, with 20 results per page. + */ findLists(props: FindListsProps) { return supertest .get('/api/lists/_find') @@ -113,7 +145,7 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .query(props.query); }, /** - * Imports a list of items from a `.txt` or `.csv` file. The maximum file size is 9 million bytes. + * Import list items from a TXT or CSV file. The maximum file size is 9 million bytes. You can import items to a new or existing list. @@ -126,6 +158,9 @@ You can import items to a new or existing list. .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .query(props.query); }, + /** + * Update specific fields of an existing list using the list ID. + */ patchList(props: PatchListProps) { return supertest .patch('/api/lists') @@ -134,6 +169,9 @@ You can import items to a new or existing list. .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .send(props.body as object); }, + /** + * Update specific fields of an existing list item using the list item ID. + */ patchListItem(props: PatchListItemProps) { return supertest .patch('/api/lists/items') @@ -142,6 +180,9 @@ You can import items to a new or existing list. .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .send(props.body as object); }, + /** + * Get the details of a list using the list ID. + */ readList(props: ReadListProps) { return supertest .get('/api/lists') @@ -150,6 +191,9 @@ You can import items to a new or existing list. .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .query(props.query); }, + /** + * Verify that `.lists` and `.items` data streams exist. + */ readListIndex() { return supertest .get('/api/lists/index') @@ -157,6 +201,9 @@ You can import items to a new or existing list. .set(ELASTIC_HTTP_VERSION_HEADER, '2023-10-31') .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); }, + /** + * Get the details of a list item. + */ readListItem(props: ReadListItemProps) { return supertest .get('/api/lists/items') @@ -172,6 +219,12 @@ You can import items to a new or existing list. .set(ELASTIC_HTTP_VERSION_HEADER, '2023-10-31') .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); }, + /** + * Update a list using the list ID. The original list is replaced, and all unspecified fields are deleted. +> info +> You cannot modify the `id` value. + + */ updateList(props: UpdateListProps) { return supertest .put('/api/lists') @@ -180,6 +233,12 @@ You can import items to a new or existing list. .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .send(props.body as object); }, + /** + * Update a list item using the list item ID. The original list item is replaced, and all unspecified fields are deleted. +> info +> You cannot modify the `id` value. + + */ updateListItem(props: UpdateListItemProps) { return supertest .put('/api/lists/items') From da2f7f6dae265e29713e5c6586c542499b03bcd6 Mon Sep 17 00:00:00 2001 From: Justin Kambic Date: Tue, 17 Sep 2024 08:08:51 -0400 Subject: [PATCH 14/58] [Synthetics] Fix issue where heatmap UI crashes on undefined histogram data (#192508) ## Summary Recently, [we fixed](https://github.com/elastic/kibana/pull/184177) an [issue](https://github.com/elastic/kibana/issues/180076) where the heatmap data on the detail and monitor history pages would not fill up. A side effect of this fix was a new regression that caused certain rarer cases to see the page crash because of an unhandled case of calling a function on a potentially-null object; our histogram data used to populate this heatmap can be `undefined` in certain cases. This patch introduces a change that will handle this case, and adds unit tests for the module in question. ### Checklist Delete any items that are not applicable to this PR. - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [ ] Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/)) - [ ] Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US)) - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server)) - [ ] This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers) ### Risk Matrix Delete this section if it is not applicable to this PR. Before closing this PR, invite QA, stakeholders, and other developers to identify risks that should be tested prior to the change/feature release. When forming the risk matrix, consider some of the following examples and how they may potentially impact the change: | Risk | Probability | Severity | Mitigation/Notes | |---------------------------|-------------|----------|-------------------------| | Multiple Spaces—unexpected behavior in non-default Kibana Space. | Low | High | Integration tests will verify that all features are still supported in non-default Kibana Space and when user switches between spaces. | | Multiple nodes—Elasticsearch polling might have race conditions when multiple Kibana nodes are polling for the same tasks. | High | Low | Tasks are idempotent, so executing them multiple times will not result in logical error, but will degrade performance. To test for this case we add plenty of unit tests around this logic and document manual testing procedure. | | Code should gracefully handle cases when feature X or plugin Y are disabled. | Medium | High | Unit tests will verify that any feature flag or plugin combination still results in our service operational. | | [See more potential risk examples](https://github.com/elastic/kibana/blob/main/RISK_MATRIX.mdx) | ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: Shahzad --- .../monitor_status_data.test.ts | 87 +++++++++++++++++++ .../monitor_status/monitor_status_data.ts | 13 ++- .../monitor_status/use_monitor_status_data.ts | 2 +- .../synthetics/state/status_heatmap/index.ts | 2 +- 4 files changed, 100 insertions(+), 4 deletions(-) create mode 100644 x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/components/monitor_details/monitor_status/monitor_status_data.test.ts diff --git a/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/components/monitor_details/monitor_status/monitor_status_data.test.ts b/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/components/monitor_details/monitor_status/monitor_status_data.test.ts new file mode 100644 index 0000000000000..488db9c2a112b --- /dev/null +++ b/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/components/monitor_details/monitor_status/monitor_status_data.test.ts @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createStatusTimeBins, getStatusEffectiveValue } from './monitor_status_data'; + +describe('createStatusTimeBins', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('should return default values when `heatmapData` is `undefined`', () => { + const timeBuckets = [ + { start: 1000, end: 2000 }, + { start: 2000, end: 3000 }, + ]; + + const result = createStatusTimeBins(timeBuckets, undefined); + + expect(result).toEqual([ + { start: 1000, end: 2000, ups: 0, downs: 0, value: 0 }, + { start: 2000, end: 3000, ups: 0, downs: 0, value: 0 }, + ]); + }); + + it('should calculate `ups` and `downs` correctly from `heatmapData`', () => { + const timeBuckets = [ + { start: 1000, end: 2000 }, + { start: 2000, end: 3000 }, + ]; + + const heatmapData = [ + { key: 1500, key_as_string: '1500', up: { value: 1 }, down: { value: 2 }, doc_count: 3 }, + { key: 2500, key_as_string: '2500', up: { value: 3 }, down: { value: 1 }, doc_count: 4 }, + ]; + + const result = createStatusTimeBins(timeBuckets, heatmapData); + + expect(result).toEqual([ + { start: 1000, end: 2000, ups: 1, downs: 2, value: getStatusEffectiveValue(1, 2) }, + { start: 2000, end: 3000, ups: 3, downs: 1, value: getStatusEffectiveValue(3, 1) }, + ]); + }); + + it('should return value 0 when ups + downs is 0', () => { + const timeBuckets = [ + { start: 1000, end: 2000 }, + { start: 2000, end: 3000 }, + ]; + + const heatmapData = [ + { key: 1500, key_as_string: '1500', up: { value: 0 }, down: { value: 0 }, doc_count: 0 }, + { key: 2500, key_as_string: '2500', up: { value: 0 }, down: { value: 0 }, doc_count: 0 }, + ]; + + const result = createStatusTimeBins(timeBuckets, heatmapData); + + expect(result).toEqual([ + { start: 1000, end: 2000, ups: 0, downs: 0, value: 0 }, + { start: 2000, end: 3000, ups: 0, downs: 0, value: 0 }, + ]); + }); + + it('should filter heatmapData correctly based on start and end values', () => { + const timeBuckets = [ + { start: 1000, end: 2000 }, + { start: 2000, end: 3000 }, + ]; + + const heatmapData = [ + { key: 500, key_as_string: '500', doc_count: 2, up: { value: 1 }, down: { value: 1 } }, + { key: 1500, key_as_string: '1500', doc_count: 5, up: { value: 2 }, down: { value: 3 } }, + { key: 2500, key_as_string: '2500', doc_count: 9, up: { value: 4 }, down: { value: 5 } }, + { key: 3500, key_as_string: '3500', doc_count: 1, up: { value: 6 }, down: { value: 7 } }, + ]; + + const result = createStatusTimeBins(timeBuckets, heatmapData); + + expect(result).toEqual([ + { start: 1000, end: 2000, ups: 2, downs: 3, value: getStatusEffectiveValue(2, 3) }, + { start: 2000, end: 3000, ups: 4, downs: 5, value: getStatusEffectiveValue(4, 5) }, + ]); + }); +}); diff --git a/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/components/monitor_details/monitor_status/monitor_status_data.ts b/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/components/monitor_details/monitor_status/monitor_status_data.ts index e5ee43aa04f8d..0a76badc574ab 100644 --- a/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/components/monitor_details/monitor_status/monitor_status_data.ts +++ b/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/components/monitor_details/monitor_status/monitor_status_data.ts @@ -114,9 +114,18 @@ export function createTimeBuckets(intervalMinutes: number, from: number, to: num export function createStatusTimeBins( timeBuckets: MonitorStatusTimeBucket[], - heatmapData: MonitorStatusHeatmapBucket[] + heatmapData?: MonitorStatusHeatmapBucket[] ): MonitorStatusTimeBin[] { return timeBuckets.map(({ start, end }) => { + if (!Array.isArray(heatmapData)) { + return { + start, + end, + ups: 0, + downs: 0, + value: 0, + }; + } const { ups, downs } = heatmapData .filter(({ key }) => key >= start && key <= end) .reduce( @@ -163,7 +172,7 @@ export function getBrushData(e: BrushEvent) { return { from, to, fromUtc, toUtc }; } -function getStatusEffectiveValue(ups: number, downs: number): number { +export function getStatusEffectiveValue(ups: number, downs: number): number { if (ups === downs) { return -0.1; } diff --git a/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/components/monitor_details/monitor_status/use_monitor_status_data.ts b/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/components/monitor_details/monitor_status/use_monitor_status_data.ts index 59d807cb3bef8..efb001f1776b7 100644 --- a/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/components/monitor_details/monitor_status/use_monitor_status_data.ts +++ b/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/components/monitor_details/monitor_status/use_monitor_status_data.ts @@ -59,7 +59,7 @@ export const useMonitorStatusData = ({ from, to, initialSizeRef }: Props) => { }, [binsAvailableByWidth, initialSizeRef]); useEffect(() => { - if (monitor?.id && location?.label && debouncedBinsCount !== null && minsPerBin !== null) { + if (monitor?.id && location?.label && debouncedBinsCount !== null && !!minsPerBin) { dispatch( quietGetMonitorStatusHeatmapAction.get({ monitorId: monitor.id, diff --git a/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/status_heatmap/index.ts b/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/status_heatmap/index.ts index 29f8a1ba87345..b01b7d0e0b918 100644 --- a/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/status_heatmap/index.ts +++ b/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/status_heatmap/index.ts @@ -18,7 +18,7 @@ import { } from './actions'; export interface MonitorStatusHeatmap { - heatmap: MonitorStatusHeatmapBucket[]; + heatmap?: MonitorStatusHeatmapBucket[]; loading: boolean; error: IHttpSerializedFetchError | null; } From 558c6fd9323fd484a95bda6078a4af913933e041 Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Tue, 17 Sep 2024 08:12:19 -0400 Subject: [PATCH 15/58] Osquery: Update exported fields reference for osquery 5.13.1 (#193095) ## Summary Update exported fields reference for osquery 5.13.1. ## Related PR - Relates https://github.com/elastic/beats/pull/40849 - Relates https://github.com/elastic/integrations/pull/11146 --- docs/osquery/exported-fields-reference.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/osquery/exported-fields-reference.asciidoc b/docs/osquery/exported-fields-reference.asciidoc index 99cb79e874754..4a09e998a1a18 100644 --- a/docs/osquery/exported-fields-reference.asciidoc +++ b/docs/osquery/exported-fields-reference.asciidoc @@ -4578,7 +4578,7 @@ For more information about osquery tables, see the https://osquery.io/schema[osq *process* - keyword, text.text -* _alf_explicit_auths.process_ - Process name explicitly allowed +* _alf_explicit_auths.process_ - Process name that is explicitly allowed * _unified_log.process_ - the name of the process that made the entry *process_being_tapped* - keyword, number.long From 193935cbf25c96ae1e6952f7233f001053e60a59 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 17 Sep 2024 08:13:54 -0400 Subject: [PATCH 16/58] [Fleet] Require AgentPolicies:All to add a fleet server (#193014) --- x-pack/plugins/fleet/common/authz.ts | 3 +- .../fleet_server_hosts_section.tsx | 2 +- .../server/services/security/security.test.ts | 53 +++++++++++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/fleet/common/authz.ts b/x-pack/plugins/fleet/common/authz.ts index 7399eb98a583b..409c5eac01d65 100644 --- a/x-pack/plugins/fleet/common/authz.ts +++ b/x-pack/plugins/fleet/common/authz.ts @@ -117,7 +117,8 @@ export const calculateAuthz = ({ allSettings: fleet.settings?.all ?? false, allAgentPolicies: fleet.agentPolicies?.all ?? false, addAgents: fleet.agents?.all ?? false, - addFleetServers: (fleet.agents?.all && fleet.settings?.all) ?? false, + addFleetServers: + (fleet.agents?.all && fleet.agentPolicies?.all && fleet.settings?.all) ?? false, // Setup is needed to access the Fleet UI setup: hasFleetAll || diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/settings_page/fleet_server_hosts_section.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/settings_page/fleet_server_hosts_section.tsx index 51d1a0f98340e..bae62ce412e35 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/settings_page/fleet_server_hosts_section.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/settings_page/fleet_server_hosts_section.tsx @@ -59,7 +59,7 @@ export const FleetServerHostsSection: React.FunctionComponent - {authz.fleet.allSettings && authz.fleet.allAgents ? ( + {authz.fleet.addFleetServers ? ( <> { expect(res.fleet.readAgents).toBe(false); }); }); + + describe('Fleet addFleetServer', () => { + beforeEach(() => { + mockSecurity.authz.mode.useRbacForRequest.mockReturnValue(true); + }); + it('should authorize user with Fleet:Agents:All Fleet:AgentsPolicies:All Fleet:Settings:All', async () => { + checkPrivileges.mockResolvedValue({ + privileges: { + kibana: [ + { + resource: 'default', + privilege: 'api:fleet-agents-all', + authorized: true, + }, + { + resource: 'default', + privilege: 'api:fleet-agent-policies-all', + authorized: true, + }, + { + resource: 'default', + privilege: 'api:fleet-settings-all', + authorized: true, + }, + ], + elasticsearch: {} as any, + }, + hasAllRequested: true, + username: 'test', + }); + const res = await getAuthzFromRequest({} as any); + expect(res.fleet.addFleetServers).toBe(true); + }); + + it('should not authorize user with only Fleet:Agents:All', async () => { + checkPrivileges.mockResolvedValue({ + privileges: { + kibana: [ + { + resource: 'default', + privilege: 'api:fleet-agents-all', + authorized: true, + }, + ], + elasticsearch: {} as any, + }, + hasAllRequested: true, + username: 'test', + }); + const res = await getAuthzFromRequest({} as any); + expect(res.fleet.addFleetServers).toBe(false); + }); + }); }); From 77fe423f7b621b2ece51ca44544c430256437802 Mon Sep 17 00:00:00 2001 From: Bharat Pasupula <123897612+bhapas@users.noreply.github.com> Date: Tue, 17 Sep 2024 14:28:01 +0200 Subject: [PATCH 17/58] [Automatic Import] Add support for handling unstructured syslog samples (#192817) ## Summary This PR handles the `unstructured` syslog samples in Automatic Import. Examples of unstructured samples would be: ``` <34>Oct 11 00:14:05 mymachine su: 'su root' failed for user on /dev/pts/8 <34>Dec 11 00:14:43 yourmachine su: 'su root' failed for someone on /dev/pts/5 <34>Apr 11 00:14:05 mymachine su: 'su root' failed for otheruser on /dev/pts/3 ``` https://github.com/user-attachments/assets/d1381ac9-4889-42cf-b3c1-d1b7a88def02 ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --- .../__jest__/fixtures/unstructured.ts | 25 ++++ .../server/graphs/kv/error.ts | 2 +- .../server/graphs/kv/header.test.ts | 2 +- .../server/graphs/kv/header.ts | 2 +- .../server/graphs/kv/validate.ts | 6 +- .../server/graphs/log_type_detection/graph.ts | 12 +- .../server/graphs/unstructured/constants.ts | 27 +++++ .../server/graphs/unstructured/error.ts | 32 +++++ .../server/graphs/unstructured/errors.test.ts | 40 +++++++ .../server/graphs/unstructured/graph.test.ts | 39 ++++++ .../server/graphs/unstructured/graph.ts | 112 ++++++++++++++++++ .../server/graphs/unstructured/index.ts | 7 ++ .../server/graphs/unstructured/prompts.ts | 105 ++++++++++++++++ .../server/graphs/unstructured/types.ts | 31 +++++ .../graphs/unstructured/unstructured.test.ts | 40 +++++++ .../graphs/unstructured/unstructured.ts | 32 +++++ .../graphs/unstructured/validate.test.ts | 71 +++++++++++ .../server/graphs/unstructured/validate.ts | 41 +++++++ .../server/routes/analyze_logs_routes.ts | 2 +- .../server/templates/processors/grok.yml.njk | 3 +- .../integration_assistant/server/types.ts | 13 ++ .../server/util/processors.ts | 4 +- 22 files changed, 633 insertions(+), 15 deletions(-) create mode 100644 x-pack/plugins/integration_assistant/__jest__/fixtures/unstructured.ts create mode 100644 x-pack/plugins/integration_assistant/server/graphs/unstructured/constants.ts create mode 100644 x-pack/plugins/integration_assistant/server/graphs/unstructured/error.ts create mode 100644 x-pack/plugins/integration_assistant/server/graphs/unstructured/errors.test.ts create mode 100644 x-pack/plugins/integration_assistant/server/graphs/unstructured/graph.test.ts create mode 100644 x-pack/plugins/integration_assistant/server/graphs/unstructured/graph.ts create mode 100644 x-pack/plugins/integration_assistant/server/graphs/unstructured/index.ts create mode 100644 x-pack/plugins/integration_assistant/server/graphs/unstructured/prompts.ts create mode 100644 x-pack/plugins/integration_assistant/server/graphs/unstructured/types.ts create mode 100644 x-pack/plugins/integration_assistant/server/graphs/unstructured/unstructured.test.ts create mode 100644 x-pack/plugins/integration_assistant/server/graphs/unstructured/unstructured.ts create mode 100644 x-pack/plugins/integration_assistant/server/graphs/unstructured/validate.test.ts create mode 100644 x-pack/plugins/integration_assistant/server/graphs/unstructured/validate.ts diff --git a/x-pack/plugins/integration_assistant/__jest__/fixtures/unstructured.ts b/x-pack/plugins/integration_assistant/__jest__/fixtures/unstructured.ts new file mode 100644 index 0000000000000..113ef4d37c073 --- /dev/null +++ b/x-pack/plugins/integration_assistant/__jest__/fixtures/unstructured.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const unstructuredLogState = { + lastExecutedChain: 'testchain', + packageName: 'testPackage', + dataStreamName: 'testDatastream', + grokPatterns: ['%{GREEDYDATA:message}'], + logSamples: ['dummy data'], + jsonSamples: ['{"message":"dummy data"}'], + finalized: false, + ecsVersion: 'testVersion', + errors: { test: 'testerror' }, + additionalProcessors: [], +}; + +export const unstructuredLogResponse = { + grok_patterns: [ + '####<%{MONTH} %{MONTHDAY}, %{YEAR} %{TIME} (?:AM|PM) %{WORD:timezone}> <%{WORD:log_level}> <%{WORD:component}> <%{DATA:hostname}> <%{DATA:server_name}> <%{DATA:thread_info}> <%{DATA:user}> <%{DATA:empty_field}> <%{DATA:empty_field2}> <%{NUMBER:timestamp}> <%{DATA:message_id}> <%{GREEDYDATA:message}>', + ], +}; diff --git a/x-pack/plugins/integration_assistant/server/graphs/kv/error.ts b/x-pack/plugins/integration_assistant/server/graphs/kv/error.ts index b6d64ee4f615d..b1b7c12a68d5a 100644 --- a/x-pack/plugins/integration_assistant/server/graphs/kv/error.ts +++ b/x-pack/plugins/integration_assistant/server/graphs/kv/error.ts @@ -33,7 +33,7 @@ export async function handleKVError({ return { kvProcessor, - lastExecutedChain: 'kv_error', + lastExecutedChain: 'kvError', }; } diff --git a/x-pack/plugins/integration_assistant/server/graphs/kv/header.test.ts b/x-pack/plugins/integration_assistant/server/graphs/kv/header.test.ts index 7991484024713..353384361d2da 100644 --- a/x-pack/plugins/integration_assistant/server/graphs/kv/header.test.ts +++ b/x-pack/plugins/integration_assistant/server/graphs/kv/header.test.ts @@ -43,6 +43,6 @@ describe('Testing kv header', () => { expect(response.grokPattern).toStrictEqual( '<%{NUMBER:priority}>%{NUMBER:version} %{GREEDYDATA:message}' ); - expect(response.lastExecutedChain).toBe('kv_header'); + expect(response.lastExecutedChain).toBe('kvHeader'); }); }); diff --git a/x-pack/plugins/integration_assistant/server/graphs/kv/header.ts b/x-pack/plugins/integration_assistant/server/graphs/kv/header.ts index 473eae1516112..36d8968ab9e67 100644 --- a/x-pack/plugins/integration_assistant/server/graphs/kv/header.ts +++ b/x-pack/plugins/integration_assistant/server/graphs/kv/header.ts @@ -26,6 +26,6 @@ export async function handleHeader({ return { grokPattern: pattern.grok_pattern, - lastExecutedChain: 'kv_header', + lastExecutedChain: 'kvHeader', }; } diff --git a/x-pack/plugins/integration_assistant/server/graphs/kv/validate.ts b/x-pack/plugins/integration_assistant/server/graphs/kv/validate.ts index 0bca2ac3fd5e4..b0601de74aa5e 100644 --- a/x-pack/plugins/integration_assistant/server/graphs/kv/validate.ts +++ b/x-pack/plugins/integration_assistant/server/graphs/kv/validate.ts @@ -41,7 +41,7 @@ export async function handleKVValidate({ )) as { pipelineResults: KVResult[]; errors: object[] }; if (errors.length > 0) { - return { errors, lastExecutedChain: 'kv_validate' }; + return { errors, lastExecutedChain: 'kvValidate' }; } // Converts JSON Object into a string and parses it as a array of JSON strings @@ -56,7 +56,7 @@ export async function handleKVValidate({ jsonSamples, additionalProcessors, errors: [], - lastExecutedChain: 'kv_validate', + lastExecutedChain: 'kvValidate', }; } @@ -65,7 +65,7 @@ export async function handleHeaderValidate({ client, }: HandleKVNodeParams): Promise> { const grokPattern = state.grokPattern; - const grokProcessor = createGrokProcessor(grokPattern); + const grokProcessor = createGrokProcessor([grokPattern]); const pipeline = { processors: grokProcessor, on_failure: [onFailure] }; const { pipelineResults, errors } = (await testPipeline(state.logSamples, pipeline, client)) as { diff --git a/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/graph.ts b/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/graph.ts index 5ef894bf64a20..b1cdecd39fe69 100644 --- a/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/graph.ts +++ b/x-pack/plugins/integration_assistant/server/graphs/log_type_detection/graph.ts @@ -14,6 +14,7 @@ import { ESProcessorItem, SamplesFormat } from '../../../common'; import { getKVGraph } from '../kv/graph'; import { LogDetectionGraphParams, LogDetectionBaseNodeParams } from './types'; import { LogFormat } from '../../constants'; +import { getUnstructuredGraph } from '../unstructured/graph'; const graphState: StateGraphArgs['channels'] = { lastExecutedChain: { @@ -90,9 +91,9 @@ function logFormatRouter({ state }: LogDetectionBaseNodeParams): string { if (state.samplesFormat.name === LogFormat.STRUCTURED) { return 'structured'; } - // if (state.samplesFormat === LogFormat.UNSTRUCTURED) { - // return 'unstructured'; - // } + if (state.samplesFormat.name === LogFormat.UNSTRUCTURED) { + return 'unstructured'; + } // if (state.samplesFormat === LogFormat.CSV) { // return 'csv'; // } @@ -109,18 +110,19 @@ export async function getLogFormatDetectionGraph({ model, client }: LogDetection handleLogFormatDetection({ state, model }) ) .addNode('handleKVGraph', await getKVGraph({ model, client })) - // .addNode('handleUnstructuredGraph', (state: LogFormatDetectionState) => getCompiledUnstructuredGraph({state, model})) + .addNode('handleUnstructuredGraph', await getUnstructuredGraph({ model, client })) // .addNode('handleCsvGraph', (state: LogFormatDetectionState) => getCompiledCsvGraph({state, model})) .addEdge(START, 'modelInput') .addEdge('modelInput', 'handleLogFormatDetection') .addEdge('handleKVGraph', 'modelOutput') + .addEdge('handleUnstructuredGraph', 'modelOutput') .addEdge('modelOutput', END) .addConditionalEdges( 'handleLogFormatDetection', (state: LogFormatDetectionState) => logFormatRouter({ state }), { structured: 'handleKVGraph', - // unstructured: 'handleUnstructuredGraph', + unstructured: 'handleUnstructuredGraph', // csv: 'handleCsvGraph', unsupported: 'modelOutput', } diff --git a/x-pack/plugins/integration_assistant/server/graphs/unstructured/constants.ts b/x-pack/plugins/integration_assistant/server/graphs/unstructured/constants.ts new file mode 100644 index 0000000000000..b0e36de9be85d --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/graphs/unstructured/constants.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const GROK_EXAMPLE_ANSWER = { + rfc: 'RFC2454', + regex: + '/(?:(d{4}[-]d{2}[-]d{2}[T]d{2}[:]d{2}[:]d{2}(?:.d{1,6})?(?:[+-]d{2}[:]d{2}|Z)?)|-)s(?:([w][wd.@-]*)|-)s(.*)$/', + grok_patterns: ['%{WORD:key1}:%{WORD:value1};%{WORD:key2}:%{WORD:value2}:%{GREEDYDATA:message}'], +}; + +export const GROK_ERROR_EXAMPLE_ANSWER = { + grok_patterns: [ + '%{TIMESTAMP:timestamp}:%{WORD:value1};%{WORD:key2}:%{WORD:value2}:%{GREEDYDATA:message}', + ], +}; + +export const onFailure = { + append: { + field: 'error.message', + value: + '{% raw %}Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}{% endraw %}', + }, +}; diff --git a/x-pack/plugins/integration_assistant/server/graphs/unstructured/error.ts b/x-pack/plugins/integration_assistant/server/graphs/unstructured/error.ts new file mode 100644 index 0000000000000..d002dd19d5439 --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/graphs/unstructured/error.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { JsonOutputParser } from '@langchain/core/output_parsers'; +import type { UnstructuredLogState } from '../../types'; +import type { HandleUnstructuredNodeParams } from './types'; +import { GROK_ERROR_PROMPT } from './prompts'; +import { GROK_ERROR_EXAMPLE_ANSWER } from './constants'; + +export async function handleUnstructuredError({ + state, + model, +}: HandleUnstructuredNodeParams): Promise> { + const outputParser = new JsonOutputParser(); + const grokErrorGraph = GROK_ERROR_PROMPT.pipe(model).pipe(outputParser); + const currentPatterns = state.grokPatterns; + + const pattern = await grokErrorGraph.invoke({ + current_pattern: JSON.stringify(currentPatterns, null, 2), + errors: JSON.stringify(state.errors, null, 2), + ex_answer: JSON.stringify(GROK_ERROR_EXAMPLE_ANSWER, null, 2), + }); + + return { + grokPatterns: pattern.grok_patterns, + lastExecutedChain: 'unstructuredError', + }; +} diff --git a/x-pack/plugins/integration_assistant/server/graphs/unstructured/errors.test.ts b/x-pack/plugins/integration_assistant/server/graphs/unstructured/errors.test.ts new file mode 100644 index 0000000000000..212b4b6255be2 --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/graphs/unstructured/errors.test.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FakeLLM } from '@langchain/core/utils/testing'; +import { handleUnstructuredError } from './error'; +import type { UnstructuredLogState } from '../../types'; +import { + unstructuredLogState, + unstructuredLogResponse, +} from '../../../__jest__/fixtures/unstructured'; +import { + ActionsClientChatOpenAI, + ActionsClientSimpleChatModel, +} from '@kbn/langchain/server/language_models'; +import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; + +const model = new FakeLLM({ + response: JSON.stringify(unstructuredLogResponse, null, 2), +}) as unknown as ActionsClientChatOpenAI | ActionsClientSimpleChatModel; + +const state: UnstructuredLogState = unstructuredLogState; + +describe('Testing unstructured error handling node', () => { + const client = { + asCurrentUser: { + ingest: { + simulate: jest.fn(), + }, + }, + } as unknown as IScopedClusterClient; + it('handleUnstructuredError()', async () => { + const response = await handleUnstructuredError({ state, model, client }); + expect(response.grokPatterns).toStrictEqual(unstructuredLogResponse.grok_patterns); + expect(response.lastExecutedChain).toBe('unstructuredError'); + }); +}); diff --git a/x-pack/plugins/integration_assistant/server/graphs/unstructured/graph.test.ts b/x-pack/plugins/integration_assistant/server/graphs/unstructured/graph.test.ts new file mode 100644 index 0000000000000..60a9bdc4329de --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/graphs/unstructured/graph.test.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + ActionsClientChatOpenAI, + ActionsClientSimpleChatModel, +} from '@kbn/langchain/server/language_models'; +import { FakeLLM } from '@langchain/core/utils/testing'; +import { getUnstructuredGraph } from './graph'; +import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; + +const model = new FakeLLM({ + response: '{"log_type": "structured"}', +}) as unknown as ActionsClientChatOpenAI | ActionsClientSimpleChatModel; + +describe('UnstructuredGraph', () => { + const client = { + asCurrentUser: { + ingest: { + simulate: jest.fn(), + }, + }, + } as unknown as IScopedClusterClient; + describe('Compiling and Running', () => { + it('Ensures that the graph compiles', async () => { + // When getUnstructuredGraph runs, langgraph compiles the graph it will error if the graph has any issues. + // Common issues for example detecting a node has no next step, or there is a infinite loop between them. + try { + await getUnstructuredGraph({ model, client }); + } catch (error) { + fail(`getUnstructuredGraph threw an error: ${error}`); + } + }); + }); +}); diff --git a/x-pack/plugins/integration_assistant/server/graphs/unstructured/graph.ts b/x-pack/plugins/integration_assistant/server/graphs/unstructured/graph.ts new file mode 100644 index 0000000000000..6048404728bfb --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/graphs/unstructured/graph.ts @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { StateGraphArgs } from '@langchain/langgraph'; +import { StateGraph, END, START } from '@langchain/langgraph'; +import type { UnstructuredLogState } from '../../types'; +import { handleUnstructured } from './unstructured'; +import type { UnstructuredGraphParams, UnstructuredBaseNodeParams } from './types'; +import { handleUnstructuredError } from './error'; +import { handleUnstructuredValidate } from './validate'; + +const graphState: StateGraphArgs['channels'] = { + lastExecutedChain: { + value: (x: string, y?: string) => y ?? x, + default: () => '', + }, + packageName: { + value: (x: string, y?: string) => y ?? x, + default: () => '', + }, + dataStreamName: { + value: (x: string, y?: string) => y ?? x, + default: () => '', + }, + logSamples: { + value: (x: string[], y?: string[]) => y ?? x, + default: () => [], + }, + grokPatterns: { + value: (x: string[], y?: string[]) => y ?? x, + default: () => [], + }, + jsonSamples: { + value: (x: string[], y?: string[]) => y ?? x, + default: () => [], + }, + finalized: { + value: (x: boolean, y?: boolean) => y ?? x, + default: () => false, + }, + errors: { + value: (x: object, y?: object) => y ?? x, + default: () => [], + }, + additionalProcessors: { + value: (x: object[], y?: object[]) => y ?? x, + default: () => [], + }, + ecsVersion: { + value: (x: string, y?: string) => y ?? x, + default: () => '', + }, +}; + +function modelInput({ state }: UnstructuredBaseNodeParams): Partial { + return { + finalized: false, + lastExecutedChain: 'modelInput', + }; +} + +function modelOutput({ state }: UnstructuredBaseNodeParams): Partial { + return { + finalized: true, + additionalProcessors: state.additionalProcessors, + lastExecutedChain: 'modelOutput', + }; +} + +function validationRouter({ state }: UnstructuredBaseNodeParams): string { + if (Object.keys(state.errors).length === 0) { + return 'modelOutput'; + } + return 'handleUnstructuredError'; +} + +export async function getUnstructuredGraph({ model, client }: UnstructuredGraphParams) { + const workflow = new StateGraph({ + channels: graphState, + }) + .addNode('modelInput', (state: UnstructuredLogState) => modelInput({ state })) + .addNode('modelOutput', (state: UnstructuredLogState) => modelOutput({ state })) + .addNode('handleUnstructuredError', (state: UnstructuredLogState) => + handleUnstructuredError({ state, model, client }) + ) + .addNode('handleUnstructured', (state: UnstructuredLogState) => + handleUnstructured({ state, model, client }) + ) + .addNode('handleUnstructuredValidate', (state: UnstructuredLogState) => + handleUnstructuredValidate({ state, model, client }) + ) + .addEdge(START, 'modelInput') + .addEdge('modelInput', 'handleUnstructured') + .addEdge('handleUnstructured', 'handleUnstructuredValidate') + .addConditionalEdges( + 'handleUnstructuredValidate', + (state: UnstructuredLogState) => validationRouter({ state }), + { + handleUnstructuredError: 'handleUnstructuredError', + modelOutput: 'modelOutput', + } + ) + .addEdge('handleUnstructuredError', 'handleUnstructuredValidate') + .addEdge('modelOutput', END); + + const compiledUnstructuredGraph = workflow.compile(); + return compiledUnstructuredGraph; +} diff --git a/x-pack/plugins/integration_assistant/server/graphs/unstructured/index.ts b/x-pack/plugins/integration_assistant/server/graphs/unstructured/index.ts new file mode 100644 index 0000000000000..8fa7bb99744ed --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/graphs/unstructured/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export { getUnstructuredGraph } from './graph'; diff --git a/x-pack/plugins/integration_assistant/server/graphs/unstructured/prompts.ts b/x-pack/plugins/integration_assistant/server/graphs/unstructured/prompts.ts new file mode 100644 index 0000000000000..5cf5c67135d53 --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/graphs/unstructured/prompts.ts @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { ChatPromptTemplate } from '@langchain/core/prompts'; + +export const GROK_MAIN_PROMPT = ChatPromptTemplate.fromMessages([ + [ + 'system', + `You are an expert in Syslogs and identifying the headers and structured body in syslog messages. Here is some context for you to reference for your task, read it carefully as you will get questions about it later: + + + {samples} + + `, + ], + [ + 'human', + `Looking at the multiple syslog samples provided in the context, You are tasked with identifying the appropriate regex and Grok pattern for a set of syslog samples. + Your goal is to accurately extract key components such as timestamps, hostnames, priority levels, process names, events, VLAN information, MAC addresses, IP addresses, STP roles, port statuses, messages and more. + + Follow these steps to help improve the grok patterns and apply it step by step: + 1. Familiarize yourself with various syslog message formats. + 2. PRI (Priority Level): Encoded in angle brackets, e.g., <134>, indicating the facility and severity. + 3. Timestamp: Use \`SYSLOGTIMESTAMP\` for RFC 3164 timestamps (e.g., Aug 10 16:34:02). Use \`TIMESTAMP_ISO8601\` for ISO 8601 (RFC 5424) timestamps. For epoch time, use \`NUMBER\`. + 4. If the timestamp could not be categorized into a predefined format, extract the date time fields separately and combine them with the format identified in the grok pattern. + 5. Make sure to identify the timezone component in the timestamp. + 6. Hostname/IP Address: The system or device that generated the message, which could be an IP address or fully qualified domain name + 7. Process Name and PID: Often included with brackets, such as sshd[1234]. + 8. VLAN information: Usually in the format of VLAN: 1234. + 9. MAC Address: The network interface MAC address. + 10. Port number: The port number on the device. + 11. Look for status codes ,interface ,log type, source ,User action, destination, protocol, etc. + 12. message: This is the free-form message text that varies widely across log entries. + + + You ALWAYS follow these guidelines when writing your response: + + - Make sure to map the remaining message part to \'message\' in grok pattern. + - Do not respond with anything except the processor as a JSON object enclosed with 3 backticks (\`), see example response above. Use strict JSON response format. + + + You are required to provide the output in the following example response format: + + + A: Please find the JSON object below: + \`\`\`json + {ex_answer} + \`\`\` + `, + ], + ['ai', 'Please find the JSON object below:'], +]); + +export const GROK_ERROR_PROMPT = ChatPromptTemplate.fromMessages([ + [ + 'system', + `You are an expert in Syslogs and identifying the headers and structured body in syslog messages. Here is some context for you to reference for your task, read it carefully as you will get questions about it later: + + +{current_pattern} + +`, + ], + [ + 'human', + `Please go through each error below, carefully review the provided current grok pattern, and resolve the most likely cause to the supplied error by returning an updated version of the current_pattern. + + +{errors} + + +Follow these steps to help improve the grok patterns and apply it step by step: + 1. Familiarize yourself with various syslog message formats. + 2. PRI (Priority Level): Encoded in angle brackets, e.g., <134>, indicating the facility and severity. + 3. Timestamp: Use \`SYSLOGTIMESTAMP\` for RFC 3164 timestamps (e.g., Aug 10 16:34:02). Use \`TIMESTAMP_ISO8601\` for ISO 8601 (RFC 5424) timestamps. For epoch time, use \`NUMBER\`. + 4. If the timestamp could not be categorized into a predefined format, extract the date time fields separately and combine them with the format identified in the grok pattern. + 5. Make sure to identify the timezone component in the timestamp. + 6. Hostname/IP Address: The system or device that generated the message, which could be an IP address or fully qualified domain name + 7. Process Name and PID: Often included with brackets, such as sshd[1234]. + 8. VLAN information: Usually in the format of VLAN: 1234. + 9. MAC Address: The network interface MAC address. + 10. Port number: The port number on the device. + 11. Look for status codes ,interface ,log type, source ,User action, destination, protocol, etc. + 12. message: This is the free-form message text that varies widely across log entries. + + You ALWAYS follow these guidelines when writing your response: + + - Make sure to map the remaining message part to \'message\' in grok pattern. + - Do not respond with anything except the processor as a JSON object enclosed with 3 backticks (\`), see example response above. Use strict JSON response format. + + + You are required to provide the output in the following example response format: + + + A: Please find the JSON object below: + \`\`\`json + {ex_answer} + \`\`\` + `, + ], + ['ai', 'Please find the JSON object below:'], +]); diff --git a/x-pack/plugins/integration_assistant/server/graphs/unstructured/types.ts b/x-pack/plugins/integration_assistant/server/graphs/unstructured/types.ts new file mode 100644 index 0000000000000..218d3856cb661 --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/graphs/unstructured/types.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; +import type { UnstructuredLogState, ChatModels } from '../../types'; + +export interface UnstructuredBaseNodeParams { + state: UnstructuredLogState; +} + +export interface UnstructuredNodeParams extends UnstructuredBaseNodeParams { + model: ChatModels; +} + +export interface UnstructuredGraphParams { + client: IScopedClusterClient; + model: ChatModels; +} + +export interface HandleUnstructuredNodeParams extends UnstructuredNodeParams { + client: IScopedClusterClient; +} + +export interface GrokResult { + [key: string]: unknown; + grok_patterns: string[]; + message: string; +} diff --git a/x-pack/plugins/integration_assistant/server/graphs/unstructured/unstructured.test.ts b/x-pack/plugins/integration_assistant/server/graphs/unstructured/unstructured.test.ts new file mode 100644 index 0000000000000..11d7107be13c0 --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/graphs/unstructured/unstructured.test.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FakeLLM } from '@langchain/core/utils/testing'; +import { handleUnstructured } from './unstructured'; +import type { UnstructuredLogState } from '../../types'; +import { + unstructuredLogState, + unstructuredLogResponse, +} from '../../../__jest__/fixtures/unstructured'; +import { + ActionsClientChatOpenAI, + ActionsClientSimpleChatModel, +} from '@kbn/langchain/server/language_models'; +import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; + +const model = new FakeLLM({ + response: JSON.stringify(unstructuredLogResponse, null, 2), +}) as unknown as ActionsClientChatOpenAI | ActionsClientSimpleChatModel; + +const state: UnstructuredLogState = unstructuredLogState; + +describe('Testing unstructured log handling node', () => { + const client = { + asCurrentUser: { + ingest: { + simulate: jest.fn(), + }, + }, + } as unknown as IScopedClusterClient; + it('handleUnstructured()', async () => { + const response = await handleUnstructured({ state, model, client }); + expect(response.grokPatterns).toStrictEqual(unstructuredLogResponse.grok_patterns); + expect(response.lastExecutedChain).toBe('handleUnstructured'); + }); +}); diff --git a/x-pack/plugins/integration_assistant/server/graphs/unstructured/unstructured.ts b/x-pack/plugins/integration_assistant/server/graphs/unstructured/unstructured.ts new file mode 100644 index 0000000000000..42186e796275f --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/graphs/unstructured/unstructured.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { JsonOutputParser } from '@langchain/core/output_parsers'; +import type { UnstructuredLogState } from '../../types'; +import { GROK_MAIN_PROMPT } from './prompts'; +import { GrokResult, HandleUnstructuredNodeParams } from './types'; +import { GROK_EXAMPLE_ANSWER } from './constants'; + +export async function handleUnstructured({ + state, + model, + client, +}: HandleUnstructuredNodeParams): Promise> { + const grokMainGraph = GROK_MAIN_PROMPT.pipe(model).pipe(new JsonOutputParser()); + + // Pick logSamples if there was no header detected. + const samples = state.logSamples; + + const pattern = (await grokMainGraph.invoke({ + samples: samples[0], + ex_answer: JSON.stringify(GROK_EXAMPLE_ANSWER, null, 2), + })) as GrokResult; + + return { + grokPatterns: pattern.grok_patterns, + lastExecutedChain: 'handleUnstructured', + }; +} diff --git a/x-pack/plugins/integration_assistant/server/graphs/unstructured/validate.test.ts b/x-pack/plugins/integration_assistant/server/graphs/unstructured/validate.test.ts new file mode 100644 index 0000000000000..493834e3220f9 --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/graphs/unstructured/validate.test.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FakeLLM } from '@langchain/core/utils/testing'; +import { handleUnstructuredValidate } from './validate'; +import type { UnstructuredLogState } from '../../types'; +import { + unstructuredLogState, + unstructuredLogResponse, +} from '../../../__jest__/fixtures/unstructured'; +import { + ActionsClientChatOpenAI, + ActionsClientSimpleChatModel, +} from '@kbn/langchain/server/language_models'; +import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; + +const model = new FakeLLM({ + response: JSON.stringify(unstructuredLogResponse, null, 2), +}) as unknown as ActionsClientChatOpenAI | ActionsClientSimpleChatModel; + +const state: UnstructuredLogState = unstructuredLogState; + +describe('Testing unstructured validation without errors', () => { + const client = { + asCurrentUser: { + ingest: { + simulate: jest + .fn() + .mockReturnValue({ docs: [{ doc: { _source: { message: 'dummy data' } } }] }), + }, + }, + } as unknown as IScopedClusterClient; + + it('handleUnstructuredValidate() without errors', async () => { + const response = await handleUnstructuredValidate({ state, model, client }); + expect(response.jsonSamples).toStrictEqual(unstructuredLogState.jsonSamples); + expect(response.additionalProcessors).toStrictEqual([ + { + grok: { + field: 'message', + patterns: unstructuredLogState.grokPatterns, + tag: 'grok_header_pattern', + }, + }, + ]); + expect(response.errors).toStrictEqual([]); + expect(response.lastExecutedChain).toBe('unstructuredValidate'); + }); +}); + +describe('Testing unstructured validation errors', () => { + const client = { + asCurrentUser: { + ingest: { + simulate: jest + .fn() + .mockReturnValue({ docs: [{ doc: { _source: { error: 'some error' } } }] }), + }, + }, + } as unknown as IScopedClusterClient; + + it('handleUnstructuredValidate() errors', async () => { + const response = await handleUnstructuredValidate({ state, model, client }); + expect(response.errors).toStrictEqual(['some error']); + expect(response.lastExecutedChain).toBe('unstructuredValidate'); + }); +}); diff --git a/x-pack/plugins/integration_assistant/server/graphs/unstructured/validate.ts b/x-pack/plugins/integration_assistant/server/graphs/unstructured/validate.ts new file mode 100644 index 0000000000000..043e38be0983f --- /dev/null +++ b/x-pack/plugins/integration_assistant/server/graphs/unstructured/validate.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { UnstructuredLogState } from '../../types'; +import type { GrokResult, HandleUnstructuredNodeParams } from './types'; +import { testPipeline } from '../../util'; +import { onFailure } from './constants'; +import { createGrokProcessor } from '../../util/processors'; + +export async function handleUnstructuredValidate({ + state, + client, +}: HandleUnstructuredNodeParams): Promise> { + const grokPatterns = state.grokPatterns; + const grokProcessor = createGrokProcessor(grokPatterns); + const pipeline = { processors: grokProcessor, on_failure: [onFailure] }; + + const { pipelineResults, errors } = (await testPipeline(state.logSamples, pipeline, client)) as { + pipelineResults: GrokResult[]; + errors: object[]; + }; + + if (errors.length > 0) { + return { errors, lastExecutedChain: 'unstructuredValidate' }; + } + + const jsonSamples: string[] = pipelineResults.map((entry) => JSON.stringify(entry)); + const additionalProcessors = state.additionalProcessors; + additionalProcessors.push(grokProcessor[0]); + + return { + jsonSamples, + additionalProcessors, + errors: [], + lastExecutedChain: 'unstructuredValidate', + }; +} diff --git a/x-pack/plugins/integration_assistant/server/routes/analyze_logs_routes.ts b/x-pack/plugins/integration_assistant/server/routes/analyze_logs_routes.ts index c0a81193a465b..29a68c4395a7c 100644 --- a/x-pack/plugins/integration_assistant/server/routes/analyze_logs_routes.ts +++ b/x-pack/plugins/integration_assistant/server/routes/analyze_logs_routes.ts @@ -81,7 +81,7 @@ export function registerAnalyzeLogsRoutes( const graph = await getLogFormatDetectionGraph({ model, client }); const graphResults = await graph.invoke(logFormatParameters, options); const graphLogFormat = graphResults.results.samplesFormat.name; - if (graphLogFormat === 'unsupported') { + if (graphLogFormat === 'unsupported' || graphLogFormat === 'csv') { return res.customError({ statusCode: 501, body: { message: `Unsupported log samples format` }, diff --git a/x-pack/plugins/integration_assistant/server/templates/processors/grok.yml.njk b/x-pack/plugins/integration_assistant/server/templates/processors/grok.yml.njk index 53ce913df0515..9b0456b134e34 100644 --- a/x-pack/plugins/integration_assistant/server/templates/processors/grok.yml.njk +++ b/x-pack/plugins/integration_assistant/server/templates/processors/grok.yml.njk @@ -1,5 +1,6 @@ - grok: field: message patterns: - - '{{ grokPattern }}' + {% for grokPattern in grokPatterns %} + - '{{ grokPattern }}'{% endfor %} tag: 'grok_header_pattern' diff --git a/x-pack/plugins/integration_assistant/server/types.ts b/x-pack/plugins/integration_assistant/server/types.ts index 0fb68b4e04572..454370a02c366 100644 --- a/x-pack/plugins/integration_assistant/server/types.ts +++ b/x-pack/plugins/integration_assistant/server/types.ts @@ -122,6 +122,19 @@ export interface KVState { ecsVersion: string; } +export interface UnstructuredLogState { + lastExecutedChain: string; + packageName: string; + dataStreamName: string; + grokPatterns: string[]; + logSamples: string[]; + jsonSamples: string[]; + finalized: boolean; + errors: object; + additionalProcessors: object[]; + ecsVersion: string; +} + export interface RelatedState { rawSamples: string[]; samples: string[]; diff --git a/x-pack/plugins/integration_assistant/server/util/processors.ts b/x-pack/plugins/integration_assistant/server/util/processors.ts index 12200f9d32db9..b2e6b1683482a 100644 --- a/x-pack/plugins/integration_assistant/server/util/processors.ts +++ b/x-pack/plugins/integration_assistant/server/util/processors.ts @@ -50,13 +50,13 @@ function createAppendProcessors(processors: SimplifiedProcessors): ESProcessorIt // The kv graph returns a simplified grok processor for header // This function takes in the grok pattern string and creates the grok processor -export function createGrokProcessor(grokPattern: string): ESProcessorItem { +export function createGrokProcessor(grokPatterns: string[]): ESProcessorItem { const templatesPath = joinPath(__dirname, '../templates/processors'); const env = new Environment(new FileSystemLoader(templatesPath), { autoescape: false, }); const template = env.getTemplate('grok.yml.njk'); - const renderedTemplate = template.render({ grokPattern }); + const renderedTemplate = template.render({ grokPatterns }); const grokProcessor = safeLoad(renderedTemplate) as ESProcessorItem; return grokProcessor; } From f4fe0bdaabaf8a8fddfc1d498743bd8a03c557e0 Mon Sep 17 00:00:00 2001 From: Carlos Crespo Date: Tue, 17 Sep 2024 15:25:34 +0200 Subject: [PATCH 18/58] [APM] Attempt to fix APMEventClient unit test (#193049) fixes https://github.com/elastic/kibana/issues/190703 ## Summary The problem seems to be on the `void` keyword used when calling `incomingRequest.abort()`. I ran a few tests changing the `setTimeout` interval with and without using the `void` keyword, and the test always passed when running without it. Co-authored-by: Elastic Machine --- .../create_es_client/create_apm_event_client/index.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.test.ts b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.test.ts index c2a5676e85c28..2239f6d8d8fb0 100644 --- a/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.test.ts +++ b/x-pack/plugins/observability_solution/apm_data_access/server/lib/helpers/create_es_client/create_apm_event_client/index.test.ts @@ -76,8 +76,9 @@ describe('APMEventClient', () => { resolve(undefined); }, 100); }); + void incomingRequest.abort(); - }, 100); + }, 200); }); expect(abortSignal?.aborted).toBe(true); From ebe4686e6c53a640d8ff2cfc60a7eeceb132812b Mon Sep 17 00:00:00 2001 From: Kerry Gallagher Date: Tue, 17 Sep 2024 14:26:54 +0100 Subject: [PATCH 19/58] [Logs] Remove AI Assistant specific log index setting (#192003) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Closes https://github.com/elastic/logs-dev/issues/167. - Removes registration of the AI Assistant specific advanced setting (`observability:aiAssistantLogsIndexPattern`). - Replaces the setting with use of the central log sources setting (`observability:logSources`). - ℹ️ Also registers the central log sources setting in serverless (this was missed previously). Due to the impact that this setting has on alerts ([Discussion](https://github.com/elastic/logs-dev/issues/170#issuecomment-2229130314)) a migration path from one setting to the other isn't really possible, as values from the AI Assistant setting could potentially cause unwanted alerts. As such these changes opt for the route that we'll do a straight swap, and document this in release notes. We will also need to do a migration on the `config` (for advanced settings) Saved Object to remove instances of the old setting / property. With the new "model version" migration model [my understanding is this should happen in a separate followup PR](https://github.com/elastic/kibana/blob/main/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_change.ts#L131). ⚠️ ~One potentially open question is whether the custom management section that the AI Assistant mounts when clicking "AI Assistant Settings" should render the log sources setting? (As this setting affects more than just the AI Assistant).~ [Resolved](https://github.com/elastic/kibana/pull/192003#issuecomment-2352727034) ![Screenshot 2024-09-05 at 11 53 31](https://github.com/user-attachments/assets/ac3816ca-9021-42f1-9a9c-4c623c6943bb) --------- Co-authored-by: Elastic Machine --- .../settings/setting_ids/index.ts | 2 - .../settings/observability_project/index.ts | 2 +- .../server/collectors/management/schema.ts | 4 - .../server/collectors/management/types.ts | 1 - src/plugins/telemetry/schema/oss_plugins.json | 6 -- .../get_log_categories/index.ts | 9 +- .../get_log_rate_analysis_for_alert/index.ts | 9 +- .../get_container_id_from_signals.ts | 13 +-- .../get_service_name_from_signals.ts | 15 ++- .../index.ts | 10 +- .../settings/indices_configuration_panel.tsx | 6 +- ...a_advanced_setting_configuration_panel.tsx | 98 +++---------------- .../log_sources_service.mocks.ts | 4 + .../services/log_sources_service/types.ts | 1 + .../services/log_sources_service/utils.ts | 11 +++ .../components/logs_sources_setting.tsx | 81 +++++++++++++++ .../public/hooks/use_log_sources.ts | 3 +- .../logs_data_access/public/index.ts | 1 + .../services/log_sources_service/index.ts | 9 +- .../services/log_sources_service/index.ts | 12 ++- .../logs_data_access/tsconfig.json | 4 +- .../common/ui_settings/settings_keys.ts | 4 +- .../observability_ai_assistant/tsconfig.json | 4 + .../kibana.jsonc | 3 +- .../server/functions/changes/index.ts | 8 +- .../server/types.ts | 2 + .../tsconfig.json | 3 +- .../common/ui_settings.ts | 19 ---- .../kibana.jsonc | 2 +- .../settings_tab/settings_tab.test.tsx | 51 +--------- .../components/settings_tab/ui_settings.tsx | 11 ++- .../tsconfig.json | 3 +- .../translations/translations/fr-FR.json | 6 +- .../translations/translations/ja-JP.json | 6 +- .../translations/translations/zh-CN.json | 6 +- .../common/ui/index.ts | 5 +- .../settings_security.spec.ts | 25 ++--- 37 files changed, 215 insertions(+), 244 deletions(-) create mode 100644 x-pack/plugins/observability_solution/logs_data_access/common/services/log_sources_service/utils.ts create mode 100644 x-pack/plugins/observability_solution/logs_data_access/public/components/logs_sources_setting.tsx diff --git a/packages/kbn-management/settings/setting_ids/index.ts b/packages/kbn-management/settings/setting_ids/index.ts index 0f79a5fff0506..611a7a0e13df5 100644 --- a/packages/kbn-management/settings/setting_ids/index.ts +++ b/packages/kbn-management/settings/setting_ids/index.ts @@ -144,8 +144,6 @@ export const OBSERVABILITY_LOGS_EXPLORER_ALLOWED_DATA_VIEWS_ID = 'observability:logsExplorer:allowedDataViews'; export const OBSERVABILITY_ENTITY_CENTRIC_EXPERIENCE = 'observability:entityCentricExperience'; export const OBSERVABILITY_LOGS_DATA_ACCESS_LOG_SOURCES_ID = 'observability:logSources'; -export const OBSERVABILITY_AI_ASSISTANT_LOGS_INDEX_PATTERN_ID = - 'observability:aiAssistantLogsIndexPattern'; export const OBSERVABILITY_AI_ASSISTANT_SIMULATED_FUNCTION_CALLING = 'observability:aiAssistantSimulatedFunctionCalling'; export const OBSERVABILITY_AI_ASSISTANT_SEARCH_CONNECTOR_INDEX_PATTERN = diff --git a/packages/serverless/settings/observability_project/index.ts b/packages/serverless/settings/observability_project/index.ts index 85f6327bf0a07..f8bb8dbe12542 100644 --- a/packages/serverless/settings/observability_project/index.ts +++ b/packages/serverless/settings/observability_project/index.ts @@ -34,8 +34,8 @@ export const OBSERVABILITY_PROJECT_SETTINGS = [ settings.OBSERVABILITY_APM_ENABLE_TABLE_SEARCH_BAR, settings.OBSERVABILITY_APM_ENABLE_SERVICE_INVENTORY_TABLE_SEARCH_BAR, settings.OBSERVABILITY_ENTITY_CENTRIC_EXPERIENCE, - settings.OBSERVABILITY_AI_ASSISTANT_LOGS_INDEX_PATTERN_ID, settings.OBSERVABILITY_AI_ASSISTANT_SIMULATED_FUNCTION_CALLING, settings.OBSERVABILITY_AI_ASSISTANT_SEARCH_CONNECTOR_INDEX_PATTERN, + settings.OBSERVABILITY_LOGS_DATA_ACCESS_LOG_SOURCES_ID, settings.OBSERVABILITY_SEARCH_EXCLUDED_DATA_TIERS, ]; diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts index 52c0df738246a..1c118620773ae 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/schema.ts @@ -484,10 +484,6 @@ export const stackManagementSchema: MakeSchemaFrom = { type: 'integer', _meta: { description: 'Non-default value of setting.' }, }, - 'observability:aiAssistantLogsIndexPattern': { - type: 'keyword', - _meta: { description: 'Non-default value of setting.' }, - }, 'observability:aiAssistantSimulatedFunctionCalling': { type: 'boolean', _meta: { description: 'Non-default value of setting.' }, diff --git a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts index 0a0ebe8ebbac6..71c692b6fdf34 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/management/types.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/management/types.ts @@ -55,7 +55,6 @@ export interface UsageStats { 'observability:apmEnableServiceInventoryTableSearchBar': boolean; 'observability:logsExplorer:allowedDataViews': string[]; 'observability:logSources': string[]; - 'observability:aiAssistantLogsIndexPattern': string; 'observability:aiAssistantSimulatedFunctionCalling': boolean; 'observability:aiAssistantSearchConnectorIndexPattern': string; 'visualization:heatmap:maxBuckets': number; diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index 77e050334803b..ce626c90c2d82 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -10319,12 +10319,6 @@ "description": "Non-default value of setting." } }, - "observability:aiAssistantLogsIndexPattern": { - "type": "keyword", - "_meta": { - "description": "Non-default value of setting." - } - }, "observability:aiAssistantSimulatedFunctionCalling": { "type": "boolean", "_meta": { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_categories/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_categories/index.ts index 41af9a4a0360d..5f36325031ccb 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_categories/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_categories/index.ts @@ -7,8 +7,7 @@ import datemath from '@elastic/datemath'; import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import type { CoreRequestHandlerContext } from '@kbn/core/server'; -import { aiAssistantLogsIndexPattern } from '@kbn/observability-ai-assistant-plugin/server'; +import { LogSourcesService } from '@kbn/logs-data-access-plugin/common/types'; import { flattenObject, KeyValuePair } from '../../../../common/utils/flatten_object'; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; import { PROCESSOR_EVENT, TRACE_ID } from '../../../../common/es_fields/apm'; @@ -26,12 +25,12 @@ export interface LogCategory { export async function getLogCategories({ apmEventClient, esClient, - coreContext, + logSourcesService, arguments: args, }: { apmEventClient: APMEventClient; esClient: ElasticsearchClient; - coreContext: Pick; + logSourcesService: LogSourcesService; arguments: { start: string; end: string; @@ -53,7 +52,7 @@ export async function getLogCategories({ Object.entries(args.entities).map(([key, value]) => ({ field: key, value })) ); - const index = await coreContext.uiSettings.client.get(aiAssistantLogsIndexPattern); + const index = await logSourcesService.getFlattenedLogSources(); const search = getTypedSearch(esClient); const query = { diff --git a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_rate_analysis_for_alert/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_rate_analysis_for_alert/index.ts index 097eff91ced14..2d367780fc9dd 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_rate_analysis_for_alert/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_rate_analysis_for_alert/index.ts @@ -6,9 +6,8 @@ */ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import type { CoreRequestHandlerContext } from '@kbn/core/server'; -import { aiAssistantLogsIndexPattern } from '@kbn/observability-ai-assistant-plugin/server'; import { fetchLogRateAnalysisForAlert } from '@kbn/aiops-log-rate-analysis/queries/fetch_log_rate_analysis_for_alert'; +import { LogSourcesService } from '@kbn/logs-data-access-plugin/common/types'; import { PROCESSOR_EVENT } from '../../../../common/es_fields/apm'; import { getShouldMatchOrNotExistFilter } from '../utils/get_should_match_or_not_exist_filter'; @@ -17,11 +16,11 @@ import { getShouldMatchOrNotExistFilter } from '../utils/get_should_match_or_not */ export async function getLogRateAnalysisForAlert({ esClient, - coreContext, + logSourcesService, arguments: args, }: { esClient: ElasticsearchClient; - coreContext: Pick; + logSourcesService: LogSourcesService; arguments: { alertStartedAt: string; alertRuleParameterTimeSize?: number; @@ -34,7 +33,7 @@ export async function getLogRateAnalysisForAlert({ }; }; }): ReturnType { - const index = await coreContext.uiSettings.client.get(aiAssistantLogsIndexPattern); + const index = await logSourcesService.getFlattenedLogSources(); const keyValueFilters = getShouldMatchOrNotExistFilter( Object.entries(args.entities).map(([key, value]) => ({ field: key, value })) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_container_id_from_signals.ts b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_container_id_from_signals.ts index 638903e813545..e7f3ace07e2a1 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_container_id_from_signals.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_container_id_from_signals.ts @@ -7,12 +7,12 @@ import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { CoreRequestHandlerContext } from '@kbn/core-http-request-handler-context-server'; -import { aiAssistantLogsIndexPattern } from '@kbn/observability-ai-assistant-plugin/common'; import { rangeQuery, typedSearch } from '@kbn/observability-plugin/server/utils/queries'; import * as t from 'io-ts'; import moment from 'moment'; import { ESSearchRequest } from '@kbn/es-types'; import { alertDetailsContextRt } from '@kbn/observability-plugin/server/services'; +import { LogSourcesService } from '@kbn/logs-data-access-plugin/common/types'; import { ApmDocumentType } from '../../../../common/document_type'; import { APMEventClient, @@ -23,11 +23,12 @@ import { RollupInterval } from '../../../../common/rollup'; export async function getContainerIdFromSignals({ query, esClient, - coreContext, + logSourcesService, apmEventClient, }: { query: t.TypeOf; esClient: ElasticsearchClient; + logSourcesService: LogSourcesService; coreContext: Pick; apmEventClient: APMEventClient; }) { @@ -66,19 +67,19 @@ export async function getContainerIdFromSignals({ return containerId; } - return getContainerIdFromLogs({ params, esClient, coreContext }); + return getContainerIdFromLogs({ params, esClient, logSourcesService }); } async function getContainerIdFromLogs({ params, esClient, - coreContext, + logSourcesService, }: { params: ESSearchRequest['body']; esClient: ElasticsearchClient; - coreContext: Pick; + logSourcesService: LogSourcesService; }) { - const index = await coreContext.uiSettings.client.get(aiAssistantLogsIndexPattern); + const index = await logSourcesService.getFlattenedLogSources(); const res = await typedSearch<{ container: { id: string } }, any>(esClient, { index, ...params, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_service_name_from_signals.ts b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_service_name_from_signals.ts index 284c286766c76..0168431d0ac4e 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_service_name_from_signals.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_service_name_from_signals.ts @@ -6,13 +6,12 @@ */ import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import { CoreRequestHandlerContext } from '@kbn/core-http-request-handler-context-server'; -import { aiAssistantLogsIndexPattern } from '@kbn/observability-ai-assistant-plugin/common'; import { rangeQuery, termQuery, typedSearch } from '@kbn/observability-plugin/server/utils/queries'; import * as t from 'io-ts'; import moment from 'moment'; import { ESSearchRequest } from '@kbn/es-types'; import { alertDetailsContextRt } from '@kbn/observability-plugin/server/services'; +import type { LogSourcesService } from '@kbn/logs-data-access-plugin/common/types'; import { ApmDocumentType } from '../../../../common/document_type'; import { APMEventClient, @@ -23,12 +22,12 @@ import { RollupInterval } from '../../../../common/rollup'; export async function getServiceNameFromSignals({ query, esClient, - coreContext, + logSourcesService, apmEventClient, }: { query: t.TypeOf; esClient: ElasticsearchClient; - coreContext: Pick; + logSourcesService: LogSourcesService; apmEventClient: APMEventClient; }) { if (query['service.name']) { @@ -75,19 +74,19 @@ export async function getServiceNameFromSignals({ return serviceName; } - return getServiceNameFromLogs({ params, esClient, coreContext }); + return getServiceNameFromLogs({ params, esClient, logSourcesService }); } async function getServiceNameFromLogs({ params, esClient, - coreContext, + logSourcesService, }: { params: ESSearchRequest['body']; esClient: ElasticsearchClient; - coreContext: Pick; + logSourcesService: LogSourcesService; }) { - const index = await coreContext.uiSettings.client.get(aiAssistantLogsIndexPattern); + const index = await logSourcesService.getFlattenedLogSources(); const res = await typedSearch<{ service: { name: string } }, any>(esClient, { index, ...params, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/index.ts index e0f3f833a5ae2..84e51675233c9 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/index.ts @@ -87,6 +87,9 @@ export const getAlertDetailsContextHandler = ( }), ]); const esClient = coreContext.elasticsearch.client.asCurrentUser; + const logSourcesService = await ( + await resourcePlugins.logsDataAccess.start() + ).services.logSourcesServiceFactory.getScopedLogSourcesService(requestContext.request); const alertStartedAt = query.alert_started_at; const serviceEnvironment = query['service.environment']; @@ -97,13 +100,14 @@ export const getAlertDetailsContextHandler = ( getServiceNameFromSignals({ query, esClient, - coreContext, + logSourcesService, apmEventClient, }), getContainerIdFromSignals({ query, esClient, coreContext, + logSourcesService, apmEventClient, }), ]); @@ -165,7 +169,7 @@ export const getAlertDetailsContextHandler = ( dataFetchers.push(async () => { const { logRateAnalysisType, significantItems } = await getLogRateAnalysisForAlert({ esClient, - coreContext, + logSourcesService, arguments: { alertStartedAt: moment(alertStartedAt).toISOString(), alertRuleParameterTimeSize: query.alert_rule_parameter_time_size @@ -203,7 +207,7 @@ export const getAlertDetailsContextHandler = ( const { logCategories, entities } = await getLogCategories({ apmEventClient, esClient, - coreContext, + logSourcesService, arguments: { start: moment(alertStartedAt).subtract(15, 'minute').toISOString(), end: alertStartedAt, diff --git a/x-pack/plugins/observability_solution/infra/public/pages/logs/settings/indices_configuration_panel.tsx b/x-pack/plugins/observability_solution/infra/public/pages/logs/settings/indices_configuration_panel.tsx index 0aad03315c8e1..8f42b4cf1bc6a 100644 --- a/x-pack/plugins/observability_solution/infra/public/pages/logs/settings/indices_configuration_panel.tsx +++ b/x-pack/plugins/observability_solution/infra/public/pages/logs/settings/indices_configuration_panel.tsx @@ -140,11 +140,7 @@ export const IndicesConfigurationPanel = React.memo<{ disabled={isReadOnly} > {isKibanaAdvancedSettingFormElement(indicesFormElement) && ( - + )} diff --git a/x-pack/plugins/observability_solution/infra/public/pages/logs/settings/kibana_advanced_setting_configuration_panel.tsx b/x-pack/plugins/observability_solution/infra/public/pages/logs/settings/kibana_advanced_setting_configuration_panel.tsx index 49636e4171f2f..a095b2825cd3e 100644 --- a/x-pack/plugins/observability_solution/infra/public/pages/logs/settings/kibana_advanced_setting_configuration_panel.tsx +++ b/x-pack/plugins/observability_solution/infra/public/pages/logs/settings/kibana_advanced_setting_configuration_panel.tsx @@ -5,33 +5,15 @@ * 2.0. */ -import { EuiDescribedFormGroup, EuiFieldText, EuiFormRow } from '@elastic/eui'; -import { FormattedMessage } from '@kbn/i18n-react'; -import React, { useMemo } from 'react'; +import React from 'react'; import { useTrackPageview } from '@kbn/observability-shared-plugin/public'; -import { LogSourcesKibanaAdvancedSettingReference } from '@kbn/logs-shared-plugin/common'; -import { ApplicationStart } from '@kbn/core-application-browser'; -import { EuiLink } from '@elastic/eui'; -import { useLogSourcesContext } from '@kbn/logs-data-access-plugin/public'; +import { + LogSourcesSettingSynchronisationInfo, + useLogSourcesContext, +} from '@kbn/logs-data-access-plugin/public'; import { useKibanaContextForPlugin } from '../../../hooks/use_kibana'; -import { FormElement } from './form_elements'; -import { getFormRowProps } from './form_field_props'; -import { FormValidationError } from './validation_errors'; -function getKibanaAdvancedSettingsHref(application: ApplicationStart) { - return application.getUrlForApp('management', { - path: `/kibana/settings?query=${encodeURIComponent('Log sources')}`, - }); -} - -export const KibanaAdvancedSettingConfigurationPanel: React.FC<{ - isLoading: boolean; - isReadOnly: boolean; - advancedSettingFormElement: FormElement< - LogSourcesKibanaAdvancedSettingReference, - FormValidationError - >; -}> = ({ isLoading, isReadOnly, advancedSettingFormElement }) => { +export const KibanaAdvancedSettingConfigurationPanel: React.FC = () => { const { services: { application }, } = useKibanaContextForPlugin(); @@ -43,71 +25,13 @@ export const KibanaAdvancedSettingConfigurationPanel: React.FC<{ delay: 15000, }); - const advancedSettingsHref = useMemo( - () => getKibanaAdvancedSettingsHref(application), - [application] - ); - const { isLoadingLogSources, combinedIndices } = useLogSourcesContext(); return ( - <> - - - - } - description={ - - - - ), - }} - /> - } - > - - } - label={ - - } - {...getFormRowProps(advancedSettingFormElement)} - > - - - - + ); }; diff --git a/x-pack/plugins/observability_solution/logs_data_access/common/services/log_sources_service/log_sources_service.mocks.ts b/x-pack/plugins/observability_solution/logs_data_access/common/services/log_sources_service/log_sources_service.mocks.ts index 3f1f8b9db5979..8073adc35d627 100644 --- a/x-pack/plugins/observability_solution/logs_data_access/common/services/log_sources_service/log_sources_service.mocks.ts +++ b/x-pack/plugins/observability_solution/logs_data_access/common/services/log_sources_service/log_sources_service.mocks.ts @@ -6,6 +6,7 @@ */ import { LogSource, LogSourcesService } from './types'; +import { flattenLogSources } from './utils'; const LOG_SOURCES: LogSource[] = [{ indexPattern: 'logs-*-*' }]; export const createLogSourcesServiceMock = ( @@ -16,6 +17,9 @@ export const createLogSourcesServiceMock = ( async getLogSources() { return Promise.resolve(sources); }, + async getFlattenedLogSources() { + return Promise.resolve(flattenLogSources(sources)); + }, async setLogSources(nextLogSources: LogSource[]) { sources = nextLogSources; return Promise.resolve(); diff --git a/x-pack/plugins/observability_solution/logs_data_access/common/services/log_sources_service/types.ts b/x-pack/plugins/observability_solution/logs_data_access/common/services/log_sources_service/types.ts index 0d4cb51051237..a2c1ae825c285 100644 --- a/x-pack/plugins/observability_solution/logs_data_access/common/services/log_sources_service/types.ts +++ b/x-pack/plugins/observability_solution/logs_data_access/common/services/log_sources_service/types.ts @@ -11,5 +11,6 @@ export interface LogSource { export interface LogSourcesService { getLogSources: () => Promise; + getFlattenedLogSources: () => Promise; setLogSources: (sources: LogSource[]) => Promise; } diff --git a/x-pack/plugins/observability_solution/logs_data_access/common/services/log_sources_service/utils.ts b/x-pack/plugins/observability_solution/logs_data_access/common/services/log_sources_service/utils.ts new file mode 100644 index 0000000000000..e6c7faa1c1140 --- /dev/null +++ b/x-pack/plugins/observability_solution/logs_data_access/common/services/log_sources_service/utils.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { LogSource } from './types'; + +export const flattenLogSources = (logSources: LogSource[]) => + logSources.map((source) => source.indexPattern).join(','); diff --git a/x-pack/plugins/observability_solution/logs_data_access/public/components/logs_sources_setting.tsx b/x-pack/plugins/observability_solution/logs_data_access/public/components/logs_sources_setting.tsx new file mode 100644 index 0000000000000..0b970ecbb2d25 --- /dev/null +++ b/x-pack/plugins/observability_solution/logs_data_access/public/components/logs_sources_setting.tsx @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { EuiDescribedFormGroup, EuiFieldText, EuiFormRow, EuiLink } from '@elastic/eui'; +import { ApplicationStart } from '@kbn/core-application-browser'; +import { FormattedMessage } from '@kbn/i18n-react'; +import React, { useMemo } from 'react'; + +export const LogSourcesSettingSynchronisationInfo: React.FC<{ + isLoading: boolean; + logSourcesValue: string; + getUrlForApp: ApplicationStart['getUrlForApp']; + title?: string; +}> = ({ isLoading, logSourcesValue, getUrlForApp, title }) => { + const advancedSettingsHref = useMemo( + () => + getUrlForApp('management', { + path: `/kibana/settings?query=${encodeURIComponent('Log sources')}`, + }), + [getUrlForApp] + ); + + return ( + <> + + {title ?? ( + + )} + + } + description={ + + + + ), + }} + /> + } + > + + } + > + + + + + ); +}; diff --git a/x-pack/plugins/observability_solution/logs_data_access/public/hooks/use_log_sources.ts b/x-pack/plugins/observability_solution/logs_data_access/public/hooks/use_log_sources.ts index ff463867a7a77..c69348c4c7275 100644 --- a/x-pack/plugins/observability_solution/logs_data_access/public/hooks/use_log_sources.ts +++ b/x-pack/plugins/observability_solution/logs_data_access/public/hooks/use_log_sources.ts @@ -8,6 +8,7 @@ import createContainer from 'constate'; import { useTrackedPromise } from '@kbn/use-tracked-promise'; import { useState, useEffect, useMemo } from 'react'; +import { flattenLogSources } from '../../common/services/log_sources_service/utils'; import { LogSource, LogSourcesService } from '../../common/services/log_sources_service/types'; export const useLogSources = ({ logSourcesService }: { logSourcesService: LogSourcesService }) => { @@ -35,7 +36,7 @@ export const useLogSources = ({ logSourcesService }: { logSourcesService: LogSou }, [getLogSources]); const combinedIndices = useMemo(() => { - return logSources.map((logSource) => logSource.indexPattern).join(','); + return flattenLogSources(logSources); }, [logSources]); return { diff --git a/x-pack/plugins/observability_solution/logs_data_access/public/index.ts b/x-pack/plugins/observability_solution/logs_data_access/public/index.ts index cc09500ef0a84..ed8facea9b91b 100644 --- a/x-pack/plugins/observability_solution/logs_data_access/public/index.ts +++ b/x-pack/plugins/observability_solution/logs_data_access/public/index.ts @@ -16,6 +16,7 @@ export type { LogsDataAccessPluginSetup, LogsDataAccessPluginStart }; import { LogsDataAccessPluginSetupDeps, LogsDataAccessPluginStartDeps } from './types'; export { LogSourcesProvider, useLogSourcesContext } from './hooks/use_log_sources'; +export { LogSourcesSettingSynchronisationInfo } from './components/logs_sources_setting'; export const plugin: PluginInitializer< LogsDataAccessPluginSetup, diff --git a/x-pack/plugins/observability_solution/logs_data_access/public/services/log_sources_service/index.ts b/x-pack/plugins/observability_solution/logs_data_access/public/services/log_sources_service/index.ts index a75bbd65c26e3..f329907f145ef 100644 --- a/x-pack/plugins/observability_solution/logs_data_access/public/services/log_sources_service/index.ts +++ b/x-pack/plugins/observability_solution/logs_data_access/public/services/log_sources_service/index.ts @@ -6,19 +6,24 @@ */ import { OBSERVABILITY_LOGS_DATA_ACCESS_LOG_SOURCES_ID } from '@kbn/management-settings-ids'; +import { flattenLogSources } from '../../../common/services/log_sources_service/utils'; import { LogSource, LogSourcesService } from '../../../common/services/log_sources_service/types'; import { RegisterServicesParams } from '../register_services'; export function createLogSourcesService(params: RegisterServicesParams): LogSourcesService { const { uiSettings } = params.deps; return { - getLogSources: async () => { + async getLogSources() { const logSources = uiSettings.get(OBSERVABILITY_LOGS_DATA_ACCESS_LOG_SOURCES_ID); return logSources.map((logSource) => ({ indexPattern: logSource, })); }, - setLogSources: async (sources: LogSource[]) => { + async getFlattenedLogSources() { + const logSources = await this.getLogSources(); + return flattenLogSources(logSources); + }, + async setLogSources(sources: LogSource[]) { await uiSettings.set( OBSERVABILITY_LOGS_DATA_ACCESS_LOG_SOURCES_ID, sources.map((source) => source.indexPattern) diff --git a/x-pack/plugins/observability_solution/logs_data_access/server/services/log_sources_service/index.ts b/x-pack/plugins/observability_solution/logs_data_access/server/services/log_sources_service/index.ts index e8907d7537932..925deb53524cd 100644 --- a/x-pack/plugins/observability_solution/logs_data_access/server/services/log_sources_service/index.ts +++ b/x-pack/plugins/observability_solution/logs_data_access/server/services/log_sources_service/index.ts @@ -8,6 +8,7 @@ import type { KibanaRequest } from '@kbn/core-http-server'; import type { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; import { OBSERVABILITY_LOGS_DATA_ACCESS_LOG_SOURCES_ID } from '@kbn/management-settings-ids'; +import { flattenLogSources } from '../../../common/services/log_sources_service/utils'; import { LogSource, LogSourcesService } from '../../../common/services/log_sources_service/types'; import { RegisterServicesParams } from '../register_services'; @@ -18,8 +19,8 @@ export function createLogSourcesServiceFactory(params: RegisterServicesParams) { ): Promise { const { uiSettings } = params.deps; const uiSettingsClient = uiSettings.asScopedToClient(savedObjectsClient); - return { - getLogSources: async () => { + const logSourcesService: LogSourcesService = { + async getLogSources() { const logSources = await uiSettingsClient.get( OBSERVABILITY_LOGS_DATA_ACCESS_LOG_SOURCES_ID ); @@ -27,13 +28,18 @@ export function createLogSourcesServiceFactory(params: RegisterServicesParams) { indexPattern: logSource, })); }, - setLogSources: async (sources: LogSource[]) => { + async getFlattenedLogSources() { + const logSources = await this.getLogSources(); + return flattenLogSources(logSources); + }, + async setLogSources(sources: LogSource[]) { return await uiSettingsClient.set( OBSERVABILITY_LOGS_DATA_ACCESS_LOG_SOURCES_ID, sources.map((source) => source.indexPattern) ); }, }; + return logSourcesService; }, async getScopedLogSourcesService(request: KibanaRequest): Promise { const { savedObjects } = params.deps; diff --git a/x-pack/plugins/observability_solution/logs_data_access/tsconfig.json b/x-pack/plugins/observability_solution/logs_data_access/tsconfig.json index fb2457edd993f..ff67c2f1c8f30 100644 --- a/x-pack/plugins/observability_solution/logs_data_access/tsconfig.json +++ b/x-pack/plugins/observability_solution/logs_data_access/tsconfig.json @@ -22,6 +22,8 @@ "@kbn/logging", "@kbn/core-saved-objects-api-server", "@kbn/es-query", - "@kbn/use-tracked-promise" + "@kbn/use-tracked-promise", + "@kbn/core-application-browser", + "@kbn/i18n-react" ] } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/common/ui_settings/settings_keys.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/common/ui_settings/settings_keys.ts index eae50d1116a8f..fdcb34f22ecde 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/common/ui_settings/settings_keys.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/common/ui_settings/settings_keys.ts @@ -5,8 +5,10 @@ * 2.0. */ +import { OBSERVABILITY_LOGS_DATA_ACCESS_LOG_SOURCES_ID } from '@kbn/management-settings-ids'; + // AI Assistant -export const aiAssistantLogsIndexPattern = 'observability:aiAssistantLogsIndexPattern'; +export const aiAssistantLogsIndexPattern = OBSERVABILITY_LOGS_DATA_ACCESS_LOG_SOURCES_ID; export const aiAssistantSimulatedFunctionCalling = 'observability:aiAssistantSimulatedFunctionCalling'; export const aiAssistantSearchConnectorIndexPattern = diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/tsconfig.json b/x-pack/plugins/observability_solution/observability_ai_assistant/tsconfig.json index aeb103e041cae..d9c747731073b 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/tsconfig.json +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/tsconfig.json @@ -42,6 +42,10 @@ "@kbn/features-plugin", "@kbn/cloud-plugin", "@kbn/serverless", + "@kbn/core-elasticsearch-server", + "@kbn/core-ui-settings-server", + "@kbn/inference-plugin", + "@kbn/management-settings-ids" ], "exclude": ["target/**/*"] } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/kibana.jsonc b/x-pack/plugins/observability_solution/observability_ai_assistant_app/kibana.jsonc index 1e02cbd1e7792..1414912d39164 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/kibana.jsonc +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/kibana.jsonc @@ -24,7 +24,8 @@ "ml", "alerting", "features", - "inference" + "inference", + "logsDataAccess" ], "requiredBundles": ["kibanaReact", "esqlDataGrid"], "optionalPlugins": ["cloud"], diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/changes/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/changes/index.ts index 987414214a17b..dc0e26ea9c777 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/changes/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/changes/index.ts @@ -7,7 +7,6 @@ import { omit, orderBy } from 'lodash'; import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query'; import type { AggregationsAutoDateHistogramAggregation } from '@elastic/elasticsearch/lib/api/types'; -import { aiAssistantLogsIndexPattern } from '@kbn/observability-ai-assistant-plugin/server'; import { createElasticsearchClient } from '../../clients/elasticsearch'; import type { FunctionRegistrationParameters } from '..'; import { @@ -25,6 +24,7 @@ export function registerChangesFunction({ logger, context: { core: corePromise }, }, + pluginsStart, }: FunctionRegistrationParameters) { functions.registerFunction( { @@ -41,7 +41,11 @@ export function registerChangesFunction({ const core = await corePromise; - const logsIndexPattern = await core.uiSettings.client.get(aiAssistantLogsIndexPattern); + const logSourcesService = + await pluginsStart.logsDataAccess.services.logSourcesServiceFactory.getLogSourcesService( + core.savedObjects.client + ); + const logsIndexPattern = await logSourcesService.getFlattenedLogSources(); const client = createElasticsearchClient({ client: core.elasticsearch.client.asCurrentUser, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/types.ts index 320459d203f9e..fc39e0b7fb24e 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/types.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/types.ts @@ -36,6 +36,7 @@ import type { CloudSetup, CloudStart } from '@kbn/cloud-plugin/server'; import type { SecurityPluginSetup, SecurityPluginStart } from '@kbn/security-plugin/server'; import type { ObservabilityPluginSetup } from '@kbn/observability-plugin/server'; import type { InferenceServerStart, InferenceServerSetup } from '@kbn/inference-plugin/server'; +import type { LogsDataAccessPluginStart } from '@kbn/logs-data-access-plugin/server'; // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface ObservabilityAIAssistantAppServerStart {} @@ -55,6 +56,7 @@ export interface ObservabilityAIAssistantAppPluginStartDependencies { cloud?: CloudStart; serverless?: ServerlessPluginStart; inference: InferenceServerStart; + logsDataAccess: LogsDataAccessPluginStart; } export interface ObservabilityAIAssistantAppPluginSetupDependencies { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/tsconfig.json b/x-pack/plugins/observability_solution/observability_ai_assistant_app/tsconfig.json index 55d965c9c37e3..84fe8f0b93911 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/tsconfig.json +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/tsconfig.json @@ -72,7 +72,8 @@ "@kbn/esql-datagrid", "@kbn/alerting-comparators", "@kbn/core-lifecycle-browser", - "@kbn/inference-plugin" + "@kbn/inference-plugin", + "@kbn/logs-data-access-plugin" ], "exclude": ["target/**/*"] } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/common/ui_settings.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/common/ui_settings.ts index f972eb5742a91..3319860de6610 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/common/ui_settings.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/common/ui_settings.ts @@ -9,30 +9,11 @@ import { schema } from '@kbn/config-schema'; import { UiSettingsParams } from '@kbn/core-ui-settings-common'; import { i18n } from '@kbn/i18n'; import { - aiAssistantLogsIndexPattern, aiAssistantSimulatedFunctionCalling, aiAssistantSearchConnectorIndexPattern, } from '@kbn/observability-ai-assistant-plugin/common'; export const uiSettings: Record = { - [aiAssistantLogsIndexPattern]: { - category: ['observability'], - name: i18n.translate( - 'xpack.observabilityAiAssistantManagement.settingsTab.h3.logIndexPatternLabel', - { defaultMessage: 'Logs index pattern' } - ), - value: 'logs-*', - description: i18n.translate( - 'xpack.observabilityAiAssistantManagement.settingsPage.logIndexPatternDescription', - { - defaultMessage: - 'Index pattern used by the AI Assistant when querying for logs. Logs are categorised and used for root cause analysis', - } - ), - schema: schema.string(), - type: 'string', - requiresPageReload: true, - }, [aiAssistantSimulatedFunctionCalling]: { category: ['observability'], name: i18n.translate( diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/kibana.jsonc b/x-pack/plugins/observability_solution/observability_ai_assistant_management/kibana.jsonc index 03d217d84e94f..ddf00c84c0ac3 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/kibana.jsonc +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/kibana.jsonc @@ -9,6 +9,6 @@ "configPath": ["xpack", "observabilityAiAssistantManagement"], "requiredPlugins": ["management", "observabilityAIAssistant", "observabilityShared"], "optionalPlugins": ["actions", "home", "serverless", "enterpriseSearch"], - "requiredBundles": ["kibanaReact"] + "requiredBundles": ["kibanaReact", "logsDataAccess"] } } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/settings_tab.test.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/settings_tab.test.tsx index 807bde0557d8d..2bed5aed37160 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/settings_tab.test.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/settings_tab.test.tsx @@ -6,11 +6,9 @@ */ import React from 'react'; -import { fireEvent, waitFor } from '@testing-library/react'; +import { fireEvent } from '@testing-library/react'; import { render } from '../../../helpers/test_helper'; import { SettingsTab } from './settings_tab'; -import { aiAssistantLogsIndexPattern } from '@kbn/observability-ai-assistant-plugin/server'; -import { uiSettings } from '../../../../common/ui_settings'; jest.mock('../../../hooks/use_app_context'); @@ -42,51 +40,4 @@ describe('SettingsTab', () => { path: '/insightsAndAlerting/triggersActionsConnectors/connectors', }); }); - - describe('allows updating the AI Assistant settings', () => { - const windowLocationReloadMock = jest.fn(); - const windowLocationOriginal = window.location; - const settingsClientSet = jest.fn(); - - beforeEach(async () => { - Object.defineProperty(window, 'location', { - value: { - reload: windowLocationReloadMock, - }, - writable: true, - }); - - const { getByTestId, container } = render(, { - coreStart: { - settings: { - client: { - set: settingsClientSet, - getAll: () => uiSettings, - }, - }, - }, - }); - - await waitFor(() => expect(container.querySelector('.euiLoadingSpinner')).toBeNull()); - - fireEvent.input(getByTestId(`management-settings-editField-${aiAssistantLogsIndexPattern}`), { - target: { value: 'observability-ai-assistant-*' }, - }); - - fireEvent.click(getByTestId('observabilityAiAssistantManagementBottomBarActionsButton')); - - await waitFor(() => expect(windowLocationReloadMock).toHaveBeenCalledTimes(1)); - }); - - afterEach(() => { - window.location = windowLocationOriginal; - }); - - it('calls the settings client with correct args', async () => { - expect(settingsClientSet).toBeCalledWith( - aiAssistantLogsIndexPattern, - 'observability-ai-assistant-*' - ); - }); - }); }); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/ui_settings.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/ui_settings.tsx index 5be8456954d64..a8c20a641f042 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/ui_settings.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/settings_tab/ui_settings.tsx @@ -17,10 +17,10 @@ import { FieldRow, FieldRowProvider } from '@kbn/management-settings-components- import { EuiSpacer } from '@elastic/eui'; import { isEmpty } from 'lodash'; import { i18n } from '@kbn/i18n'; +import { LogSourcesSettingSynchronisationInfo } from '@kbn/logs-data-access-plugin/public'; import { useKibana } from '../../../hooks/use_kibana'; const settingsKeys = [ - aiAssistantLogsIndexPattern, aiAssistantSimulatedFunctionCalling, aiAssistantSearchConnectorIndexPattern, aiAssistantPreferredAIAssistantType, @@ -31,7 +31,7 @@ export function UISettings() { docLinks, settings, notifications, - application: { capabilities }, + application: { capabilities, getUrlForApp }, } = useKibana().services; const { fields, handleFieldChange, unsavedChanges, saveAll, isSaving, cleanUnsavedChanges } = @@ -84,6 +84,13 @@ export function UISettings() { ); })} + + + {!isEmpty(unsavedChanges) && ( > Fonctionnalités.", "xpack.observabilityAiAssistantManagement.settingsPage.simulatedFunctionCallingDescription": "[version d'évaluation technique] Utilisez l'appel de fonction simulé. L’appel de fonction simulé ne nécessite pas la prise en charge de l'API pour les fonctions ou les outils, mais il peut réduire les performances. L'appel de fonction simulé est actuellement toujours activé pour les connecteurs non-OpenAI, indépendamment de ce paramètre.", "xpack.observabilityAiAssistantManagement.settingsPage.simulatedFunctionCallingLabel": "Simuler un appel de fonction", - "xpack.observabilityAiAssistantManagement.settingsTab.h3.logIndexPatternLabel": "Modèle d'indexation des logs", "xpack.observabilityAiAssistantManagement.settingsTab.h3.searchConnectorIndexPatternLabel": "Modèle d'indexation de connecteur de recherche", "xpack.observabilityAiAssistantManagement.span.expandRowLabel": "Développer la ligne", "xpack.observabilityLogsExplorer.alertsPopover.buttonLabel": "Alertes", @@ -47895,4 +47893,4 @@ "xpack.watcher.watchEdit.thresholdWatchExpression.aggType.fieldIsRequiredValidationMessage": "Ce champ est requis.", "xpack.watcher.watcherDescription": "Détectez les modifications survenant dans vos données en créant, gérant et monitorant des alertes." } -} \ No newline at end of file +} diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 9280f3b99b867..f2237c38b3275 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -766,8 +766,8 @@ "core.euiDataGridCell.focusTrapEnterPrompt": "このセルの内容を操作するには、Enterキーを押してください。", "core.euiDataGridCell.position": "{columnId}, 列{col}, 行{row}", "core.euiDataGridCellActions.expandButtonTitle": "クリックするか enter を押すと、セルのコンテンツとインタラクトできます。", - "core.euiDataGridHeaderCell.actionsPopoverScreenReaderText": "列アクションのリストを移動するには、Tabまたは上下矢印キーを押します。", "core.euiDataGridHeaderCell.actionsButtonAriaLabel": "{title}。クリックすると、列ヘッダーアクションが表示されます", + "core.euiDataGridHeaderCell.actionsPopoverScreenReaderText": "列アクションのリストを移動するには、Tabまたは上下矢印キーを押します。", "core.euiDataGridHeaderCell.sortedByAscendingFirst": "{columnId}の昇順で並べ替え", "core.euiDataGridHeaderCell.sortedByAscendingMultiple": "、{columnId}の昇順で並べ替え", "core.euiDataGridHeaderCell.sortedByAscendingSingle": "昇順で並べ替えます", @@ -32954,7 +32954,6 @@ "xpack.observabilityAiAssistantManagement.settingsPage.goToFeatureControlsButtonLabel": "スペースに移動", "xpack.observabilityAiAssistantManagement.settingsPage.h2.settingsLabel": "設定", "xpack.observabilityAiAssistantManagement.settingsPage.knowledgeBaseLabel": "ナレッジベース", - "xpack.observabilityAiAssistantManagement.settingsPage.logIndexPatternDescription": "ログのクエリを実行するときにAI Assistantによって使用されるインデックスパターン。ログは分類され、根本原因分析で使用されます。", "xpack.observabilityAiAssistantManagement.settingsPage.searchConnector": "検索コネクター", "xpack.observabilityAiAssistantManagement.settingsPage.searchConnectorIndexPatternDescription": "検索コネクターインデックス（ナレッジベースの一部）をクエリするときに、AI Assistantによって使用されるインデックスパターン。デフォルトでは、すべての検索コネクターのインデックスに対してクエリが実行されます。", "xpack.observabilityAiAssistantManagement.settingsPage.settingsLabel": "設定", @@ -32962,7 +32961,6 @@ "xpack.observabilityAiAssistantManagement.settingsPage.showAIAssistantDescriptionLabel": "オブザーバビリティアプリでAI Assistantボタンと状況に応じたインサイトのオン/オフを切り替えるには、［スペース］>［］>［機能］でAI Assistant機能のチェックをオンまたはオフにします。", "xpack.observabilityAiAssistantManagement.settingsPage.simulatedFunctionCallingDescription": "[technical preview] シミュレートされた関数呼び出しを使用します。シミュレートされた関数呼び出しでは、関数またはツールのAPIサポートは必要ありませんが、パフォーマンスが低下する可能性があります。現在、シミュレートされた関数呼び出しは、この設定に関係なく、非OpenAIコネクターで常に有効です。", "xpack.observabilityAiAssistantManagement.settingsPage.simulatedFunctionCallingLabel": "関数呼び出しをシミュレート", - "xpack.observabilityAiAssistantManagement.settingsTab.h3.logIndexPatternLabel": "ログインデックスパターン", "xpack.observabilityAiAssistantManagement.settingsTab.h3.searchConnectorIndexPatternLabel": "検索コネクターインデックスパターン", "xpack.observabilityAiAssistantManagement.span.expandRowLabel": "行を展開", "xpack.observabilityLogsExplorer.alertsPopover.buttonLabel": "アラート", @@ -47878,4 +47876,4 @@ "xpack.watcher.watchEdit.thresholdWatchExpression.aggType.fieldIsRequiredValidationMessage": "フィールドを選択してください。", "xpack.watcher.watcherDescription": "アラートの作成、管理、監視によりデータへの変更を検知します。" } -} \ No newline at end of file +} diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 08826a2d5775e..c57a42b464836 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -764,8 +764,8 @@ "core.euiDataGridCell.focusTrapEnterPrompt": "按 Enter 键与此单元格的内容进行交互。", "core.euiDataGridCell.position": "{columnId}，列 {col}，行 {row}", "core.euiDataGridCellActions.expandButtonTitle": "单击或按 Enter 键以便与单元格内容进行交互", - "core.euiDataGridHeaderCell.actionsPopoverScreenReaderText": "要在列操作列表中导航，请按 Tab 键或向上和向下箭头键。", "core.euiDataGridHeaderCell.actionsButtonAriaLabel": "{title}。单击以查看列标题操作", + "core.euiDataGridHeaderCell.actionsPopoverScreenReaderText": "要在列操作列表中导航，请按 Tab 键或向上和向下箭头键。", "core.euiDataGridHeaderCell.sortedByAscendingFirst": "按 {columnId} 排序，升序", "core.euiDataGridHeaderCell.sortedByAscendingMultiple": "，然后按 {columnId} 排序，升序", "core.euiDataGridHeaderCell.sortedByAscendingSingle": "已升序", @@ -32994,7 +32994,6 @@ "xpack.observabilityAiAssistantManagement.settingsPage.goToFeatureControlsButtonLabel": "前往工作区", "xpack.observabilityAiAssistantManagement.settingsPage.h2.settingsLabel": "设置", "xpack.observabilityAiAssistantManagement.settingsPage.knowledgeBaseLabel": "知识库", - "xpack.observabilityAiAssistantManagement.settingsPage.logIndexPatternDescription": "AI 助手查询日志时使用的索引模式。日志将进行归类并用于根本原因分析", "xpack.observabilityAiAssistantManagement.settingsPage.searchConnector": "搜索连接器", "xpack.observabilityAiAssistantManagement.settingsPage.searchConnectorIndexPatternDescription": "查询搜索连接器索引（知识库的一部分）时 AI 助手使用的索引模式。默认情况下，将查询每个搜索连接器的索引", "xpack.observabilityAiAssistantManagement.settingsPage.settingsLabel": "设置", @@ -33002,7 +33001,6 @@ "xpack.observabilityAiAssistantManagement.settingsPage.showAIAssistantDescriptionLabel": "通过在“工作区 > > 功能”中选中或取消选中 AI 助手功能，在 Observability 应用中打开或关闭 AI 助手按钮和上下文洞察。", "xpack.observabilityAiAssistantManagement.settingsPage.simulatedFunctionCallingDescription": "[技术预览] 使用模拟函数调用。模拟函数调用不需要函数或工具的 API 支持，但可能会降低性能。无论此设置如何，当前会始终对非 OpenAI 连接器启用模拟函数调用。", "xpack.observabilityAiAssistantManagement.settingsPage.simulatedFunctionCallingLabel": "模拟函数调用", - "xpack.observabilityAiAssistantManagement.settingsTab.h3.logIndexPatternLabel": "日志索引模式", "xpack.observabilityAiAssistantManagement.settingsTab.h3.searchConnectorIndexPatternLabel": "搜索连接器索引模式", "xpack.observabilityAiAssistantManagement.span.expandRowLabel": "展开行", "xpack.observabilityLogsExplorer.alertsPopover.buttonLabel": "告警", @@ -47929,4 +47927,4 @@ "xpack.watcher.watchEdit.thresholdWatchExpression.aggType.fieldIsRequiredValidationMessage": "此字段必填。", "xpack.watcher.watcherDescription": "通过创建、管理和监测警报来检测数据中的更改。" } -} \ No newline at end of file +} diff --git a/x-pack/test/observability_ai_assistant_functional/common/ui/index.ts b/x-pack/test/observability_ai_assistant_functional/common/ui/index.ts index a5d2802dfbcc5..2c6852988cde5 100644 --- a/x-pack/test/observability_ai_assistant_functional/common/ui/index.ts +++ b/x-pack/test/observability_ai_assistant_functional/common/ui/index.ts @@ -57,8 +57,9 @@ const pages = { settings: { settingsPage: 'aiAssistantSettingsPage', managementLink: 'aiAssistantManagementSelection', - logsIndexPatternInput: - 'management-settings-editField-observability:aiAssistantLogsIndexPattern', + logsIndexPatternInput: 'management-settings-editField-observability:logSources', + searchConnectorIndexPatternInput: + 'management-settings-editField-observability:aiAssistantSearchConnectorIndexPattern', saveButton: 'observabilityAiAssistantManagementBottomBarActionsButton', aiAssistantCard: 'aiAssistantSelectionPageObservabilityCard', }, diff --git a/x-pack/test/observability_ai_assistant_functional/tests/feature_controls/settings_security.spec.ts b/x-pack/test/observability_ai_assistant_functional/tests/feature_controls/settings_security.spec.ts index ef87707c62a2c..96f2ff4b00f7f 100644 --- a/x-pack/test/observability_ai_assistant_functional/tests/feature_controls/settings_security.spec.ts +++ b/x-pack/test/observability_ai_assistant_functional/tests/feature_controls/settings_security.spec.ts @@ -58,20 +58,21 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { await testSubjects.existOrFail(ui.pages.settings.settingsPage); }); it('allows updating of an advanced setting', async () => { - const testLogsIndexPattern = 'my-logs-index-pattern'; - const logsIndexPatternInput = await testSubjects.find( - ui.pages.settings.logsIndexPatternInput + const testSearchConnectorIndexPattern = 'my-logs-index-pattern'; + const searchConnectorIndexPatternInput = await testSubjects.find( + ui.pages.settings.searchConnectorIndexPatternInput ); - await logsIndexPatternInput.clearValue(); - await logsIndexPatternInput.type(testLogsIndexPattern); + await searchConnectorIndexPatternInput.clearValue(); + await searchConnectorIndexPatternInput.type(testSearchConnectorIndexPattern); const saveButton = await testSubjects.find(ui.pages.settings.saveButton); await saveButton.click(); await browser.refresh(); - const logsIndexPatternInputValue = await logsIndexPatternInput.getAttribute('value'); - expect(logsIndexPatternInputValue).to.be(testLogsIndexPattern); + const searchConnectorIndexPatternInputValue = + await searchConnectorIndexPatternInput.getAttribute('value'); + expect(searchConnectorIndexPatternInputValue).to.be(testSearchConnectorIndexPattern); // reset the value - await logsIndexPatternInput.clearValue(); - await logsIndexPatternInput.type('logs-*'); + await searchConnectorIndexPatternInput.clearValue(); + await searchConnectorIndexPatternInput.type('logs-*'); await saveButton.click(); }); }); @@ -114,10 +115,10 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { await testSubjects.existOrFail(ui.pages.settings.settingsPage); }); it('has disabled inputs', async () => { - const logsIndexPatternInput = await testSubjects.find( - ui.pages.settings.logsIndexPatternInput + const searchConnectorIndexPatternInput = await testSubjects.find( + ui.pages.settings.searchConnectorIndexPatternInput ); - expect(await logsIndexPatternInput.getAttribute('disabled')).to.be('true'); + expect(await searchConnectorIndexPatternInput.getAttribute('disabled')).to.be('true'); }); }); describe('observabilityAIAssistant privilege with no aiAssistantManagementSelection privilege', () => { From b3a1e5fb8f3bd6a05943ca3646186a0e3567cfe9 Mon Sep 17 00:00:00 2001 From: Ignacio Rivas Date: Tue, 17 Sep 2024 16:35:20 +0300 Subject: [PATCH 20/58] [Console] UX Improvements for phase 2 (#190698) --- .buildkite/ftr_platform_stateful_configs.yml | 3 +- packages/kbn-monaco/index.ts | 1 + packages/kbn-monaco/src/console/index.ts | 2 + .../kbn-monaco/src/console/output_parser.js | 401 +++++++++ .../src/console/output_parser.test.ts | 40 + packages/kbn-monaco/src/console/types.ts | 10 + .../constants/autocomplete_definitions.ts | 2 + src/plugins/console/common/constants/index.ts | 1 + .../application/components/console_menu.tsx | 16 +- .../components/console_tour_step.tsx | 64 ++ .../components/editor_content_spinner.tsx | 6 +- .../application/components/editor_example.tsx | 91 -- .../application/components/help_panel.tsx | 163 ---- .../application/components/help_popover.tsx | 136 +++ .../public/application/components/index.ts | 46 +- .../components/output_panel_empty_state.tsx | 57 ++ .../application/components/settings/index.ts | 31 + .../components/settings/settings_editor.tsx | 371 ++++++++ .../components/settings/settings_form_row.tsx | 33 + .../components/settings/settings_group.tsx | 37 + .../index.ts => components/settings/types.ts} | 3 +- .../application/components/settings_modal.tsx | 391 -------- .../shortcuts_popover}/index.ts | 3 +- .../components/shortcuts_popover/keys.tsx | 69 ++ .../shortcuts_popover/shortcut_line.tsx | 65 ++ .../shortcuts_popover/shortcuts_popover.tsx | 114 +++ .../application/components/top_nav_menu.tsx | 21 +- .../application/components/variables/index.ts | 24 +- .../application/components/variables/types.ts | 14 + .../application/components/variables/utils.ts | 30 +- .../components/variables/variables_editor.tsx | 255 ++++++ .../variables/variables_editor_form.tsx | 184 ++++ .../components/variables/variables_flyout.tsx | 215 ----- .../application/components/welcome_panel.tsx | 167 ---- .../application/containers/config/config.tsx | 47 + .../{editor/utilities => config}/index.ts | 2 +- .../containers/{ => config}/settings.tsx | 19 +- .../containers/{ => config}/variables.tsx | 19 +- .../console_history/console_history.tsx | 242 ----- .../console_history/history_viewer.tsx | 61 -- .../components/context_menu/context_menu.tsx | 36 +- .../components/context_menu/index.ts | 0 .../context_menu/language_selector_modal.tsx | 2 +- .../editor/{monaco => }/components/index.ts | 0 .../application/containers/editor/editor.tsx | 350 ++++++-- .../editor/{monaco => }/hooks/index.ts | 0 .../hooks/use_register_keyboard_commands.ts | 0 .../hooks/use_resize_checker_utils.ts | 0 .../hooks/use_set_initial_value.ts | 12 +- .../hooks/use_setup_autocomplete_polling.ts | 2 +- .../{monaco => }/hooks/use_setup_autosave.ts | 2 +- .../application/containers/editor/index.ts | 1 - .../console_editor/apply_editor_settings.ts | 27 - .../console_editor/editor.test.mock.tsx | 50 -- .../legacy/console_editor/editor.test.tsx | 100 --- .../editor/legacy/console_editor/editor.tsx | 343 ------- .../legacy/console_editor/editor_output.tsx | 125 --- .../console_editor/keyboard_shortcuts.ts | 92 -- .../editor/legacy/console_menu_actions.ts | 39 - .../containers/editor/legacy/index.ts | 11 - .../subscribe_console_resize_checker.ts | 28 - .../containers/editor/monaco/index.ts | 11 - .../editor/{monaco => }/monaco_editor.tsx | 63 +- .../monaco_editor_actions_provider.test.ts | 4 +- .../monaco_editor_actions_provider.ts | 123 ++- .../{monaco => }/monaco_editor_output.tsx | 88 +- .../monaco_editor_output_actions_provider.ts | 185 ++++ .../monaco_editor_suggestion_provider.ts | 0 .../containers/editor/{monaco => }/types.ts | 0 .../utils/autocomplete_utils.test.ts | 4 +- .../{monaco => }/utils/autocomplete_utils.ts | 6 +- .../editor/{monaco => }/utils/constants.ts | 0 .../editor/{monaco => }/utils/index.ts | 7 + ...convert_mapbox_vector_tile_to_json.test.ts | 0 .../convert_mapbox_vector_tile_to_json.ts | 0 .../mapbox_vector_tile/index.ts | 0 .../mapbox_vector_tile/response.pbf | Bin .../{utilities => utils}/output_data.ts | 0 .../{monaco => }/utils/requests_utils.test.ts | 2 +- .../{monaco => }/utils/requests_utils.ts | 6 +- .../status_code_decoration_utils.test.ts | 2 +- .../utils/status_code_decoration_utils.ts | 2 +- .../{monaco => }/utils/tokens_utils.test.ts | 0 .../editor/{monaco => }/utils/tokens_utils.ts | 0 .../containers/embeddable/console_wrapper.tsx | 5 +- .../embeddable/embeddable_console.tsx | 2 - .../containers/history/history.tsx | 306 +++++++ .../containers/history/history_empty.tsx | 61 ++ .../history_viewer_monaco.tsx | 5 +- .../{console_history => history}/index.ts | 2 +- .../public/application/containers/index.ts | 1 - .../application/containers/main/constants.ts | 30 + .../main/get_console_tour_step_props.tsx | 82 ++ .../containers/main/get_top_nav.ts | 82 +- .../containers/main/get_tour_steps.tsx | 122 +++ .../application/containers/main/i18n.ts | 43 + .../containers/main/import_confirm_modal.tsx | 67 ++ .../application/containers/main/index.ts | 1 + .../application/containers/main/main.tsx | 387 ++++++-- .../containers/main/nav_icon_button.tsx | 38 + .../__snapshots__/split_panel.test.tsx.snap | 101 --- .../containers/split_panel/panel.tsx | 47 - .../split_panel/panel_container.tsx | 140 --- .../containers/split_panel/resizer.tsx | 35 - .../split_panel/split_panel.test.tsx | 74 -- .../editor_context/editor_registry.ts | 2 +- .../public/application/contexts/index.ts | 2 - .../contexts/services_context.mock.ts | 3 +- .../application/contexts/services_context.tsx | 3 +- .../application/contexts/split_panel/index.ts | 11 - .../split_panel/split_panel_context.tsx | 30 - .../restore_request_from_history_to_monaco.ts | 2 +- .../use_restore_request_from_history.ts | 21 +- .../use_send_current_request/send_request.ts | 5 +- .../application/hooks/use_set_input_editor.ts | 2 +- .../console/public/application/index.tsx | 35 +- .../format_request.ts} | 22 +- .../console/public/application/lib/index.ts | 1 + .../public/application/stores/editor.ts | 38 +- .../public/application/stores/request.ts | 7 + .../console/public/lib/autocomplete/types.ts | 2 +- .../lib/autocomplete_entities/data_stream.ts | 11 +- src/plugins/console/public/plugin.ts | 9 +- src/plugins/console/public/services/api.ts | 2 +- .../console/public/services/autocomplete.ts | 12 + .../console/public/services/storage.ts | 2 +- src/plugins/console/public/shared_imports.ts | 13 + src/plugins/console/public/styles/_app.scss | 91 +- src/plugins/console/public/types/common.ts | 10 + src/plugins/console/public/types/config.ts | 3 - .../public/types/embeddable_console.ts | 1 - src/plugins/console/server/config.ts | 2 - .../services/spec_definitions_service.test.ts | 1 + .../services/spec_definitions_service.ts | 13 +- src/plugins/console/tsconfig.json | 6 +- src/plugins/dev_tools/public/application.tsx | 9 +- src/plugins/dev_tools/public/dev_tool.ts | 1 + test/accessibility/apps/console.ts | 2 +- .../apps/console/{ace => }/_autocomplete.ts | 156 ++-- .../apps/console/{monaco => }/_comments.ts | 16 +- .../apps/console/{monaco => }/_console.ts | 124 ++- .../apps/console/{monaco => }/_console_ccs.ts | 12 +- .../console/{monaco => }/_context_menu.ts | 32 +- .../apps/console/_misc_console_behavior.ts | 262 ++++++ .../apps/console/_onboarding_tour.ts | 107 +++ test/functional/apps/console/_output_panel.ts | 85 ++ .../apps/console/{ace => }/_settings.ts | 18 +- .../apps/console/{monaco => }/_text_input.ts | 55 +- .../apps/console/{monaco => }/_variables.ts | 32 +- .../apps/console/{ace => }/_vector_tile.ts | 8 +- .../apps/console/{monaco => }/_xjson.ts | 54 +- test/functional/apps/console/ace/_comments.ts | 232 ----- test/functional/apps/console/ace/_console.ts | 210 ----- .../apps/console/ace/_console_ccs.ts | 52 -- .../apps/console/ace/_context_menu.ts | 104 --- .../console/ace/_misc_console_behavior.ts | 142 --- .../apps/console/ace/_text_input.ts | 102 --- .../functional/apps/console/ace/_variables.ts | 75 -- test/functional/apps/console/ace/_xjson.ts | 189 ---- test/functional/apps/console/ace/config.ts | 28 - .../apps/console/{monaco => }/config.ts | 4 +- .../apps/console/{ace => }/index.ts | 4 +- .../apps/console/monaco/_autocomplete.ts | 386 -------- .../console/monaco/_misc_console_behavior.ts | 183 ---- .../apps/console/monaco/_settings.ts | 42 - .../apps/console/monaco/_vector_tile.ts | 52 -- test/functional/apps/console/monaco/index.ts | 35 - .../controls/common/config.ts | 5 - .../controls/common/control_group_chaining.ts | 13 +- .../controls/options_list/config.ts | 5 - ...ptions_list_allow_expensive_queries_off.ts | 5 +- .../options_list_dashboard_interaction.ts | 12 +- test/functional/config.ccs.ts | 2 +- test/functional/firefox/console.config.ts | 2 +- test/functional/page_objects/console_page.ts | 837 ++++++------------ .../page_objects/embedded_console.ts | 3 + .../test_suites/core_plugins/rendering.ts | 1 - .../translations/translations/fr-FR.json | 61 -- .../translations/translations/ja-JP.json | 61 -- .../translations/translations/zh-CN.json | 61 -- .../apps/dev_tools/embedded_console.ts | 1 + .../apps/reporting_management/config.ts | 5 - .../reporting_management/report_listing.ts | 9 +- .../page_objects/embedded_console.ts | 3 + .../apps/ccs/ccs_console.js | 6 +- .../test_suites/common/console/console.ts | 32 +- .../test_suites/search/embedded_console.ts | 1 + 187 files changed, 5167 insertions(+), 5943 deletions(-) create mode 100644 packages/kbn-monaco/src/console/output_parser.js create mode 100644 packages/kbn-monaco/src/console/output_parser.test.ts create mode 100644 src/plugins/console/public/application/components/console_tour_step.tsx delete mode 100644 src/plugins/console/public/application/components/editor_example.tsx delete mode 100644 src/plugins/console/public/application/components/help_panel.tsx create mode 100644 src/plugins/console/public/application/components/help_popover.tsx create mode 100644 src/plugins/console/public/application/components/output_panel_empty_state.tsx create mode 100644 src/plugins/console/public/application/components/settings/index.ts create mode 100644 src/plugins/console/public/application/components/settings/settings_editor.tsx create mode 100644 src/plugins/console/public/application/components/settings/settings_form_row.tsx create mode 100644 src/plugins/console/public/application/components/settings/settings_group.tsx rename src/plugins/console/public/application/{containers/split_panel/index.ts => components/settings/types.ts} (84%) delete mode 100644 src/plugins/console/public/application/components/settings_modal.tsx rename src/plugins/console/public/application/{containers/editor/legacy/console_editor => components/shortcuts_popover}/index.ts (84%) create mode 100644 src/plugins/console/public/application/components/shortcuts_popover/keys.tsx create mode 100644 src/plugins/console/public/application/components/shortcuts_popover/shortcut_line.tsx create mode 100644 src/plugins/console/public/application/components/shortcuts_popover/shortcuts_popover.tsx create mode 100644 src/plugins/console/public/application/components/variables/types.ts create mode 100644 src/plugins/console/public/application/components/variables/variables_editor.tsx create mode 100644 src/plugins/console/public/application/components/variables/variables_editor_form.tsx delete mode 100644 src/plugins/console/public/application/components/variables/variables_flyout.tsx delete mode 100644 src/plugins/console/public/application/components/welcome_panel.tsx create mode 100644 src/plugins/console/public/application/containers/config/config.tsx rename src/plugins/console/public/application/containers/{editor/utilities => config}/index.ts (93%) rename src/plugins/console/public/application/containers/{ => config}/settings.tsx (88%) rename src/plugins/console/public/application/containers/{ => config}/variables.tsx (66%) delete mode 100644 src/plugins/console/public/application/containers/console_history/console_history.tsx delete mode 100644 src/plugins/console/public/application/containers/console_history/history_viewer.tsx rename src/plugins/console/public/application/containers/editor/{monaco => }/components/context_menu/context_menu.tsx (90%) rename src/plugins/console/public/application/containers/editor/{monaco => }/components/context_menu/index.ts (100%) rename src/plugins/console/public/application/containers/editor/{monaco => }/components/context_menu/language_selector_modal.tsx (98%) rename src/plugins/console/public/application/containers/editor/{monaco => }/components/index.ts (100%) rename src/plugins/console/public/application/containers/editor/{monaco => }/hooks/index.ts (100%) rename src/plugins/console/public/application/containers/editor/{monaco => }/hooks/use_register_keyboard_commands.ts (100%) rename src/plugins/console/public/application/containers/editor/{monaco => }/hooks/use_resize_checker_utils.ts (100%) rename src/plugins/console/public/application/containers/editor/{monaco => }/hooks/use_set_initial_value.ts (91%) rename src/plugins/console/public/application/containers/editor/{monaco => }/hooks/use_setup_autocomplete_polling.ts (94%) rename src/plugins/console/public/application/containers/editor/{monaco => }/hooks/use_setup_autosave.ts (96%) delete mode 100644 src/plugins/console/public/application/containers/editor/legacy/console_editor/apply_editor_settings.ts delete mode 100644 src/plugins/console/public/application/containers/editor/legacy/console_editor/editor.test.mock.tsx delete mode 100644 src/plugins/console/public/application/containers/editor/legacy/console_editor/editor.test.tsx delete mode 100644 src/plugins/console/public/application/containers/editor/legacy/console_editor/editor.tsx delete mode 100644 src/plugins/console/public/application/containers/editor/legacy/console_editor/editor_output.tsx delete mode 100644 src/plugins/console/public/application/containers/editor/legacy/console_editor/keyboard_shortcuts.ts delete mode 100644 src/plugins/console/public/application/containers/editor/legacy/console_menu_actions.ts delete mode 100644 src/plugins/console/public/application/containers/editor/legacy/index.ts delete mode 100644 src/plugins/console/public/application/containers/editor/legacy/subscribe_console_resize_checker.ts delete mode 100644 src/plugins/console/public/application/containers/editor/monaco/index.ts rename src/plugins/console/public/application/containers/editor/{monaco => }/monaco_editor.tsx (79%) rename src/plugins/console/public/application/containers/editor/{monaco => }/monaco_editor_actions_provider.test.ts (99%) rename src/plugins/console/public/application/containers/editor/{monaco => }/monaco_editor_actions_provider.ts (86%) rename src/plugins/console/public/application/containers/editor/{monaco => }/monaco_editor_output.tsx (60%) create mode 100644 src/plugins/console/public/application/containers/editor/monaco_editor_output_actions_provider.ts rename src/plugins/console/public/application/containers/editor/{monaco => }/monaco_editor_suggestion_provider.ts (100%) rename src/plugins/console/public/application/containers/editor/{monaco => }/types.ts (100%) rename src/plugins/console/public/application/containers/editor/{monaco => }/utils/autocomplete_utils.test.ts (98%) rename src/plugins/console/public/application/containers/editor/{monaco => }/utils/autocomplete_utils.ts (99%) rename src/plugins/console/public/application/containers/editor/{monaco => }/utils/constants.ts (100%) rename src/plugins/console/public/application/containers/editor/{monaco => }/utils/index.ts (86%) rename src/plugins/console/public/application/containers/editor/{legacy/console_editor => utils}/mapbox_vector_tile/convert_mapbox_vector_tile_to_json.test.ts (100%) rename src/plugins/console/public/application/containers/editor/{legacy/console_editor => utils}/mapbox_vector_tile/convert_mapbox_vector_tile_to_json.ts (100%) rename src/plugins/console/public/application/containers/editor/{legacy/console_editor => utils}/mapbox_vector_tile/index.ts (100%) rename src/plugins/console/public/application/containers/editor/{legacy/console_editor => utils}/mapbox_vector_tile/response.pbf (100%) rename src/plugins/console/public/application/containers/editor/{utilities => utils}/output_data.ts (100%) rename src/plugins/console/public/application/containers/editor/{monaco => }/utils/requests_utils.test.ts (99%) rename src/plugins/console/public/application/containers/editor/{monaco => }/utils/requests_utils.ts (98%) rename src/plugins/console/public/application/containers/editor/{monaco => }/utils/status_code_decoration_utils.test.ts (98%) rename src/plugins/console/public/application/containers/editor/{monaco => }/utils/status_code_decoration_utils.ts (95%) rename src/plugins/console/public/application/containers/editor/{monaco => }/utils/tokens_utils.test.ts (100%) rename src/plugins/console/public/application/containers/editor/{monaco => }/utils/tokens_utils.ts (100%) create mode 100644 src/plugins/console/public/application/containers/history/history.tsx create mode 100644 src/plugins/console/public/application/containers/history/history_empty.tsx rename src/plugins/console/public/application/containers/{console_history => history}/history_viewer_monaco.tsx (94%) rename src/plugins/console/public/application/containers/{console_history => history}/index.ts (90%) create mode 100644 src/plugins/console/public/application/containers/main/constants.ts create mode 100644 src/plugins/console/public/application/containers/main/get_console_tour_step_props.tsx create mode 100644 src/plugins/console/public/application/containers/main/get_tour_steps.tsx create mode 100644 src/plugins/console/public/application/containers/main/i18n.ts create mode 100644 src/plugins/console/public/application/containers/main/import_confirm_modal.tsx create mode 100644 src/plugins/console/public/application/containers/main/nav_icon_button.tsx delete mode 100644 src/plugins/console/public/application/containers/split_panel/__snapshots__/split_panel.test.tsx.snap delete mode 100644 src/plugins/console/public/application/containers/split_panel/panel.tsx delete mode 100644 src/plugins/console/public/application/containers/split_panel/panel_container.tsx delete mode 100644 src/plugins/console/public/application/containers/split_panel/resizer.tsx delete mode 100644 src/plugins/console/public/application/containers/split_panel/split_panel.test.tsx delete mode 100644 src/plugins/console/public/application/contexts/split_panel/index.ts delete mode 100644 src/plugins/console/public/application/contexts/split_panel/split_panel_context.tsx rename src/plugins/console/public/application/{contexts/split_panel/split_panel_registry.ts => lib/format_request.ts} (57%) rename test/functional/apps/console/{ace => }/_autocomplete.ts (74%) rename test/functional/apps/console/{monaco => }/_comments.ts (88%) rename test/functional/apps/console/{monaco => }/_console.ts (50%) rename test/functional/apps/console/{monaco => }/_console_ccs.ts (84%) rename test/functional/apps/console/{monaco => }/_context_menu.ts (85%) create mode 100644 test/functional/apps/console/_misc_console_behavior.ts create mode 100644 test/functional/apps/console/_onboarding_tour.ts create mode 100644 test/functional/apps/console/_output_panel.ts rename test/functional/apps/console/{ace => }/_settings.ts (71%) rename test/functional/apps/console/{monaco => }/_text_input.ts (64%) rename test/functional/apps/console/{monaco => }/_variables.ts (74%) rename test/functional/apps/console/{ace => }/_vector_tile.ts (88%) rename test/functional/apps/console/{monaco => }/_xjson.ts (62%) delete mode 100644 test/functional/apps/console/ace/_comments.ts delete mode 100644 test/functional/apps/console/ace/_console.ts delete mode 100644 test/functional/apps/console/ace/_console_ccs.ts delete mode 100644 test/functional/apps/console/ace/_context_menu.ts delete mode 100644 test/functional/apps/console/ace/_misc_console_behavior.ts delete mode 100644 test/functional/apps/console/ace/_text_input.ts delete mode 100644 test/functional/apps/console/ace/_variables.ts delete mode 100644 test/functional/apps/console/ace/_xjson.ts delete mode 100644 test/functional/apps/console/ace/config.ts rename test/functional/apps/console/{monaco => }/config.ts (88%) rename test/functional/apps/console/{ace => }/index.ts (88%) delete mode 100644 test/functional/apps/console/monaco/_autocomplete.ts delete mode 100644 test/functional/apps/console/monaco/_misc_console_behavior.ts delete mode 100644 test/functional/apps/console/monaco/_settings.ts delete mode 100644 test/functional/apps/console/monaco/_vector_tile.ts delete mode 100644 test/functional/apps/console/monaco/index.ts diff --git a/.buildkite/ftr_platform_stateful_configs.yml b/.buildkite/ftr_platform_stateful_configs.yml index fc46fa24f257f..02d6355c212bd 100644 --- a/.buildkite/ftr_platform_stateful_configs.yml +++ b/.buildkite/ftr_platform_stateful_configs.yml @@ -44,8 +44,7 @@ enabled: - test/api_integration/config.js - test/examples/config.js - test/functional/apps/bundles/config.ts - - test/functional/apps/console/monaco/config.ts - - test/functional/apps/console/ace/config.ts + - test/functional/apps/console/config.ts - test/functional/apps/context/config.ts - test/functional/apps/dashboard_elements/controls/common/config.ts - test/functional/apps/dashboard_elements/controls/options_list/config.ts diff --git a/packages/kbn-monaco/index.ts b/packages/kbn-monaco/index.ts index 795664b60e7b7..ba8b0edb68e1a 100644 --- a/packages/kbn-monaco/index.ts +++ b/packages/kbn-monaco/index.ts @@ -39,6 +39,7 @@ export { CONSOLE_THEME_ID, getParsedRequestsProvider, ConsoleParsedRequestsProvider, + createOutputParser, } from './src/console'; export type { ParsedRequest } from './src/console'; diff --git a/packages/kbn-monaco/src/console/index.ts b/packages/kbn-monaco/src/console/index.ts index 6b26c6262f568..cf36505b27759 100644 --- a/packages/kbn-monaco/src/console/index.ts +++ b/packages/kbn-monaco/src/console/index.ts @@ -43,3 +43,5 @@ export const ConsoleOutputLang: LangModuleType = { export type { ParsedRequest } from './types'; export { getParsedRequestsProvider } from './language'; export { ConsoleParsedRequestsProvider } from './console_parsed_requests_provider'; + +export { createOutputParser } from './output_parser'; diff --git a/packages/kbn-monaco/src/console/output_parser.js b/packages/kbn-monaco/src/console/output_parser.js new file mode 100644 index 0000000000000..8601cf764055e --- /dev/null +++ b/packages/kbn-monaco/src/console/output_parser.js @@ -0,0 +1,401 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +/* eslint-disable prettier/prettier,prefer-const,no-throw-literal,camelcase,@typescript-eslint/no-shadow,one-var,object-shorthand,eqeqeq */ + +export const createOutputParser = () => { + let at, // The index of the current character + ch, // The current character + escapee = { + '"': '"', + '\\': '\\', + '/': '/', + b: '\b', + f: '\f', + n: '\n', + r: '\r', + t: '\t', + }, + text, + errors, + addError = function (text) { + errors.push({ text: text, offset: at }); + }, + responses, + responseStartOffset, + responseEndOffset, + getLastResponse = function() { + return responses.length > 0 ? responses.pop() : {}; + }, + addResponseStart = function() { + responseStartOffset = at - 1; + responses.push({ startOffset: responseStartOffset }); + }, + addResponseData = function(data) { + const lastResponse = getLastResponse(); + const dataArray = lastResponse.data || []; + dataArray.push(data); + lastResponse.data = dataArray; + responses.push(lastResponse); + responseEndOffset = at - 1; + }, + addResponseEnd = function() { + const lastResponse = getLastResponse(); + lastResponse.endOffset = responseEndOffset; + responses.push(lastResponse); + }, + error = function (m) { + throw { + name: 'SyntaxError', + message: m, + at: at, + text: text, + }; + }, + reset = function (newAt) { + ch = text.charAt(newAt); + at = newAt + 1; + }, + next = function (c) { + if (c && c !== ch) { + error('Expected \'' + c + '\' instead of \'' + ch + '\''); + } + + ch = text.charAt(at); + at += 1; + return ch; + }, + nextUpTo = function (upTo, errorMessage) { + let currentAt = at, + i = text.indexOf(upTo, currentAt); + if (i < 0) { + error(errorMessage || 'Expected \'' + upTo + '\''); + } + reset(i + upTo.length); + return text.substring(currentAt, i); + }, + peek = function (offset) { + return text.charAt(at + offset); + }, + number = function () { + let number, + string = ''; + + if (ch === '-') { + string = '-'; + next('-'); + } + while (ch >= '0' && ch <= '9') { + string += ch; + next(); + } + if (ch === '.') { + string += '.'; + while (next() && ch >= '0' && ch <= '9') { + string += ch; + } + } + if (ch === 'e' || ch === 'E') { + string += ch; + next(); + if (ch === '-' || ch === '+') { + string += ch; + next(); + } + while (ch >= '0' && ch <= '9') { + string += ch; + next(); + } + } + number = +string; + if (isNaN(number)) { + error('Bad number'); + } else { + return number; + } + }, + string = function () { + let hex, + i, + string = '', + uffff; + + if (ch === '"') { + // If the current and the next characters are equal to "", empty string or start of triple quoted strings + if (peek(0) === '"' && peek(1) === '"') { + // literal + next('"'); + next('"'); + return nextUpTo('"""', 'failed to find closing \'"""\''); + } else { + while (next()) { + if (ch === '"') { + next(); + return string; + } else if (ch === '\\') { + next(); + if (ch === 'u') { + uffff = 0; + for (i = 0; i < 4; i += 1) { + hex = parseInt(next(), 16); + if (!isFinite(hex)) { + break; + } + uffff = uffff * 16 + hex; + } + string += String.fromCharCode(uffff); + } else if (typeof escapee[ch] === 'string') { + string += escapee[ch]; + } else { + break; + } + } else { + string += ch; + } + } + } + } + error('Bad string'); + }, + white = function () { + while (ch) { + // Skip whitespace. + while (ch && ch <= ' ') { + next(); + } + // if the current char in iteration is '#' or the char and the next char is equal to '//' + // we are on the single line comment + if (ch === '#' || ch === '/' && peek(0) === '/') { + // Until we are on the new line, skip to the next char + while (ch && ch !== '\n') { + next(); + } + } else if (ch === '/' && peek(0) === '*') { + // If the chars starts with '/*', we are on the multiline comment + next(); + next(); + while (ch && !(ch === '*' && peek(0) === '/')) { + // Until we have closing tags '*/', skip to the next char + next(); + } + if (ch) { + next(); + next(); + } + } else break; + } + }, + strictWhite = function () { + while (ch && (ch == ' ' || ch == '\t')) { + next(); + } + }, + newLine = function () { + if (ch == '\n') next(); + }, + word = function () { + switch (ch) { + case 't': + next('t'); + next('r'); + next('u'); + next('e'); + return true; + case 'f': + next('f'); + next('a'); + next('l'); + next('s'); + next('e'); + return false; + case 'n': + next('n'); + next('u'); + next('l'); + next('l'); + return null; + } + error('Unexpected \'' + ch + '\''); + }, + value, // Place holder for the value function. + array = function () { + const array = []; + + if (ch === '[') { + next('['); + white(); + if (ch === ']') { + next(']'); + return array; // empty array + } + while (ch) { + array.push(value()); + white(); + if (ch === ']') { + next(']'); + return array; + } + next(','); + white(); + } + } + error('Bad array'); + }, + object = function () { + let key, + object = {}; + + if (ch === '{') { + next('{'); + white(); + if (ch === '}') { + next('}'); + return object; // empty object + } + while (ch) { + key = string(); + white(); + next(':'); + if (Object.hasOwnProperty.call(object, key)) { + error('Duplicate key "' + key + '"'); + } + object[key] = value(); + white(); + if (ch === '}') { + next('}'); + return object; + } + next(','); + white(); + } + } + error('Bad object'); + }; + + value = function () { + white(); + switch (ch) { + case '{': + return object(); + case '[': + return array(); + case '"': + return string(); + case '-': + return number(); + default: + return ch >= '0' && ch <= '9' ? number() : word(); + } + }; + + let response = function () { + white(); + addResponseStart(); + // it can be an object + if (ch == '{') { + const parsedObject = object(); + addResponseData(parsedObject); + // but it could also be an array of objects + } else if (ch == '[') { + const parsedArray = array(); + parsedArray.forEach(item => { + if (typeof item === 'object') { + addResponseData(item); + } else { + error('Array elements must be objects'); + } + }); + } else { + error('Invalid input'); + } + // multi doc response + strictWhite(); // advance to one new line + newLine(); + strictWhite(); + while (ch == '{') { + // another object + const parsedObject = object(); + addResponseData(parsedObject); + strictWhite(); + newLine(); + strictWhite(); + } + addResponseEnd(); + }, + comment = function () { + while (ch == '#') { + while (ch && ch !== '\n') { + next(); + } + white(); + } + }, + multi_response = function () { + while (ch && ch != '') { + white(); + if (!ch) { + continue; + } + try { + comment(); + white(); + if (!ch) { + continue; + } + response(); + white(); + } catch (e) { + addError(e.message); + // snap + const substring = text.substr(at); + const nextMatch = substring.search(/[#{]/); + if (nextMatch < 1) return; + reset(at + nextMatch); + } + } + }; + + return function (source, reviver) { + let result; + + text = source; + at = 0; + errors = []; + responses = []; + next(); + multi_response(); + white(); + if (ch) { + addError('Syntax error'); + } + + result = { errors, responses }; + + return typeof reviver === 'function' + ? (function walk(holder, key) { + let k, + v, + value = holder[key]; + if (value && typeof value === 'object') { + for (k in value) { + if (Object.hasOwnProperty.call(value, k)) { + v = walk(value, k); + if (v !== undefined) { + value[k] = v; + } else { + delete value[k]; + } + } + } + } + return reviver.call(holder, key, value); + }({ '': result }, '')) + : result; + }; +} diff --git a/packages/kbn-monaco/src/console/output_parser.test.ts b/packages/kbn-monaco/src/console/output_parser.test.ts new file mode 100644 index 0000000000000..47ec0bbeb65e4 --- /dev/null +++ b/packages/kbn-monaco/src/console/output_parser.test.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { createOutputParser } from './output_parser'; +import { ConsoleOutputParserResult } from './types'; + +const parser = createOutputParser(); +describe('console output parser', () => { + it('returns errors if input is not correct', () => { + const input = 'x'; + const parserResult = parser(input) as ConsoleOutputParserResult; + + expect(parserResult.responses.length).toBe(1); + // the parser should generate an invalid input error + expect(parserResult.errors).toContainEqual({ text: 'Invalid input', offset: 1 }); + }); + + it('returns parsed responses if the input is correct', () => { + const input = `# 1: GET /my-index/_doc/0 \n { "_index": "my-index" }`; + const { responses, errors } = parser(input) as ConsoleOutputParserResult; + expect(responses.length).toBe(1); + expect(errors.length).toBe(0); + const { data } = responses[0]; + + const expected = [{ _index: 'my-index' }]; + expect(data).toEqual(expected); + }); + + it('parses several responses', () => { + const input = `# 1: GET /my-index/_doc/0 \n { "_index": "my-index" } \n # 2: GET /my-index/_doc/1 \n { "_index": "my-index" }`; + const { responses } = parser(input) as ConsoleOutputParserResult; + expect(responses.length).toBe(2); + }); +}); diff --git a/packages/kbn-monaco/src/console/types.ts b/packages/kbn-monaco/src/console/types.ts index 6c7573eabdb2c..a024e4696f8cd 100644 --- a/packages/kbn-monaco/src/console/types.ts +++ b/packages/kbn-monaco/src/console/types.ts @@ -21,6 +21,16 @@ export interface ConsoleParserResult { requests: ParsedRequest[]; } +export interface ConsoleOutputParsedResponse { + startOffset: number; + endOffset?: number; + data?: Array>; +} +export interface ConsoleOutputParserResult { + errors: ErrorAnnotation[]; + responses: ConsoleOutputParsedResponse[]; +} + export interface ConsoleWorkerDefinition { getParserResult: (modelUri: string) => ConsoleParserResult | undefined; } diff --git a/src/plugins/console/common/constants/autocomplete_definitions.ts b/src/plugins/console/common/constants/autocomplete_definitions.ts index b2ef4f1375419..0ab69c1fa9528 100644 --- a/src/plugins/console/common/constants/autocomplete_definitions.ts +++ b/src/plugins/console/common/constants/autocomplete_definitions.ts @@ -17,3 +17,5 @@ export const AUTOCOMPLETE_DEFINITIONS_FOLDER = resolve( export const GENERATED_SUBFOLDER = 'generated'; export const OVERRIDES_SUBFOLDER = 'overrides'; export const MANUAL_SUBFOLDER = 'manual'; + +export const API_DOCS_LINK = 'https://www.elastic.co/docs/api'; diff --git a/src/plugins/console/common/constants/index.ts b/src/plugins/console/common/constants/index.ts index ea572db743ef4..a00bcebcf38cc 100644 --- a/src/plugins/console/common/constants/index.ts +++ b/src/plugins/console/common/constants/index.ts @@ -15,6 +15,7 @@ export { GENERATED_SUBFOLDER, OVERRIDES_SUBFOLDER, MANUAL_SUBFOLDER, + API_DOCS_LINK, } from './autocomplete_definitions'; export { DEFAULT_INPUT_VALUE } from './editor_input'; export { DEFAULT_LANGUAGE, AVAILABLE_LANGUAGES } from './copy_as'; diff --git a/src/plugins/console/public/application/components/console_menu.tsx b/src/plugins/console/public/application/components/console_menu.tsx index 3ed1d67c3602b..4dee3ff06df0a 100644 --- a/src/plugins/console/public/application/components/console_menu.tsx +++ b/src/plugins/console/public/application/components/console_menu.tsx @@ -11,13 +11,7 @@ import React, { Component } from 'react'; import { NotificationsSetup } from '@kbn/core/public'; -import { - EuiIcon, - EuiContextMenuPanel, - EuiContextMenuItem, - EuiPopover, - EuiLink, -} from '@elastic/eui'; +import { EuiContextMenuPanel, EuiContextMenuItem, EuiPopover, EuiButtonIcon } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n-react'; import { i18n } from '@kbn/i18n'; @@ -115,15 +109,15 @@ export class ConsoleMenu extends Component { render() { const button = ( - - - + iconType="boxesVertical" + iconSize="s" + /> ); const items = [ diff --git a/src/plugins/console/public/application/components/console_tour_step.tsx b/src/plugins/console/public/application/components/console_tour_step.tsx new file mode 100644 index 0000000000000..578d590bfff4a --- /dev/null +++ b/src/plugins/console/public/application/components/console_tour_step.tsx @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import React, { ReactNode, ReactElement } from 'react'; +import { EuiTourStep, PopoverAnchorPosition } from '@elastic/eui'; + +export interface ConsoleTourStepProps { + step: number; + stepsTotal: number; + isStepOpen: boolean; + title: ReactNode; + content: ReactNode; + onFinish: () => void; + footerAction: ReactNode | ReactNode[]; + dataTestSubj: string; + anchorPosition: string; + maxWidth: number; + css?: any; +} + +interface Props { + tourStepProps: ConsoleTourStepProps; + children: ReactNode & ReactElement; +} + +export const ConsoleTourStep = ({ tourStepProps, children }: Props) => { + const { + step, + isStepOpen, + stepsTotal, + title, + content, + onFinish, + footerAction, + dataTestSubj, + anchorPosition, + maxWidth, + css, + } = tourStepProps; + + return ( + + {children} + + ); +}; diff --git a/src/plugins/console/public/application/components/editor_content_spinner.tsx b/src/plugins/console/public/application/components/editor_content_spinner.tsx index eecd9aebf67cc..a4d0ccc98b76c 100644 --- a/src/plugins/console/public/application/components/editor_content_spinner.tsx +++ b/src/plugins/console/public/application/components/editor_content_spinner.tsx @@ -8,12 +8,12 @@ */ import React, { FunctionComponent } from 'react'; -import { EuiSkeletonText, EuiPageSection } from '@elastic/eui'; +import { EuiLoadingSpinner, EuiPageSection } from '@elastic/eui'; export const EditorContentSpinner: FunctionComponent = () => { return ( - - + + ); }; diff --git a/src/plugins/console/public/application/components/editor_example.tsx b/src/plugins/console/public/application/components/editor_example.tsx deleted file mode 100644 index 6a5ab6333c3b5..0000000000000 --- a/src/plugins/console/public/application/components/editor_example.tsx +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import { EuiScreenReaderOnly, withEuiTheme } from '@elastic/eui'; -import type { WithEuiThemeProps } from '@elastic/eui'; -import { i18n } from '@kbn/i18n'; -import React, { useEffect, useRef } from 'react'; -import { createReadOnlyAceEditor, CustomAceEditor } from '../models/sense_editor'; -// @ts-ignore -import { Mode as InputMode } from '../models/legacy_core_editor/mode/input'; -import { Mode as OutputMode } from '../models/legacy_core_editor/mode/output'; - -interface EditorExampleProps { - panel: string; - example?: string; - theme: WithEuiThemeProps['theme']; - linesOfExampleCode?: number; - mode?: string; -} - -const exampleText = ` -GET _search -{ - "query": { - "match_all": {} - } -} -`; - -const EditorExample = ({ - panel, - example, - theme, - linesOfExampleCode = 6, - mode = 'input', -}: EditorExampleProps) => { - const inputId = `help-example-${panel}-input`; - const wrapperDivRef = useRef(null); - const editorRef = useRef(); - - useEffect(() => { - if (wrapperDivRef.current) { - editorRef.current = createReadOnlyAceEditor(wrapperDivRef.current); - - const editor = editorRef.current; - const editorMode = mode === 'input' ? new InputMode() : new OutputMode(); - editor.update((example || exampleText).trim(), editorMode); - editor.session.setUseWorker(false); - editor.setHighlightActiveLine(false); - - const textareaElement = wrapperDivRef.current.querySelector('textarea'); - if (textareaElement) { - textareaElement.setAttribute('id', inputId); - textareaElement.setAttribute('readonly', 'true'); - } - } - - return () => { - if (editorRef.current) { - editorRef.current.destroy(); - } - }; - }, [example, inputId, mode]); - - const wrapperDivStyle = { - height: `${parseInt(theme.euiTheme.size.base, 10) * linesOfExampleCode}px`, - margin: `${theme.euiTheme.size.base} 0`, - }; - - return ( - <> - - - -
- - ); -}; - -// eslint-disable-next-line import/no-default-export -export default withEuiTheme(EditorExample); diff --git a/src/plugins/console/public/application/components/help_panel.tsx b/src/plugins/console/public/application/components/help_panel.tsx deleted file mode 100644 index 30a356e27002e..0000000000000 --- a/src/plugins/console/public/application/components/help_panel.tsx +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import React from 'react'; -import { FormattedMessage } from '@kbn/i18n-react'; -import { - EuiText, - EuiFlyout, - EuiFlyoutHeader, - EuiFlyoutBody, - EuiTitle, - EuiSpacer, - EuiLink, -} from '@elastic/eui'; -import EditorExample from './editor_example'; -import { useServicesContext } from '../contexts'; - -interface Props { - onClose: () => void; -} - -export function HelpPanel(props: Props) { - const { docLinks } = useServicesContext(); - - return ( - - - -

- -

- - - - -

- -

-

- -

-

- - Console - - ), - queryDsl: ( - - Query DSL - - ), - }} - /> -

- -

- -

- -
-
Ctrl/Cmd + I
-
- -
-
Ctrl/Cmd + /
-
- -
-
Ctrl + Space
-
- -
-
Ctrl/Cmd + Enter
-
- -
-
Ctrl/Cmd + Up/Down
-
- -
-
Ctrl/Cmd + Alt + L
-
- -
-
Ctrl/Cmd + Option + 0
-
- -
-
Down arrow
-
- -
-
Enter/Tab
-
- -
-
Ctrl/Cmd + L
-
- -
-
Esc
-
- -
-
- - - - ); -} diff --git a/src/plugins/console/public/application/components/help_popover.tsx b/src/plugins/console/public/application/components/help_popover.tsx new file mode 100644 index 0000000000000..16e9465d4d388 --- /dev/null +++ b/src/plugins/console/public/application/components/help_popover.tsx @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import React from 'react'; +import { i18n } from '@kbn/i18n'; +import { + EuiPopover, + EuiTitle, + EuiText, + EuiFlexGroup, + EuiFlexItem, + EuiButtonIcon, + EuiSpacer, +} from '@elastic/eui'; +import { useServicesContext } from '../contexts'; + +interface HelpPopoverProps { + button: any; + isOpen: boolean; + closePopover: () => void; + resetTour: () => void; +} + +export const HelpPopover = ({ button, isOpen, closePopover, resetTour }: HelpPopoverProps) => { + const { docLinks } = useServicesContext(); + + return ( + + +

+ {i18n.translate('console.helpPopover.title', { + defaultMessage: 'Elastic Console', + })} +

+ + + + + +

+ {i18n.translate('console.helpPopover.description', { + defaultMessage: + 'Console is an interactive UI for calling Elasticsearch and Kibana APIs and viewing their responses. Search your data, manage settings, and more, using Query DSL and REST API syntax.', + })} +

+ + + + + + + + +

+ {i18n.translate('console.helpPopover.aboutConsoleLabel', { + defaultMessage: 'About Console', + })} +

+ + + + + + + + + + +

+ {i18n.translate('console.helpPopover.aboutQueryDSLLabel', { + defaultMessage: 'About Query DSL', + })} +

+ + + + + + + + + + +

+ {i18n.translate('console.helpPopover.rerunTourLabel', { + defaultMessage: 'Re-run feature tour', + })} +

+ + + + + + + + + ); +}; diff --git a/src/plugins/console/public/application/components/index.ts b/src/plugins/console/public/application/components/index.ts index e091fb5f2f8a5..111778d8fa776 100644 --- a/src/plugins/console/public/application/components/index.ts +++ b/src/plugins/console/public/application/components/index.ts @@ -7,50 +7,14 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import React from 'react'; -import { withSuspense } from '@kbn/shared-ux-utility'; - export { NetworkRequestStatusBar } from './network_request_status_bar'; export { SomethingWentWrongCallout } from './something_went_wrong_callout'; export type { TopNavMenuItem } from './top_nav_menu'; export { TopNavMenu } from './top_nav_menu'; export { ConsoleMenu } from './console_menu'; -export { WelcomePanel } from './welcome_panel'; -export type { AutocompleteOptions } from './settings_modal'; -export { HelpPanel } from './help_panel'; export { EditorContentSpinner } from './editor_content_spinner'; -export type { DevToolsVariable } from './variables'; - -/** - * The Lazily-loaded `DevToolsSettingsModal` component. Consumers should use `React.Suspense` or - * the withSuspense` HOC to load this component. - */ -export const DevToolsSettingsModalLazy = React.lazy(() => - import('./settings_modal').then(({ DevToolsSettingsModal }) => ({ - default: DevToolsSettingsModal, - })) -); - -/** - * A `DevToolsSettingsModal` component that is wrapped by the `withSuspense` HOC. This component can - * be used directly by consumers and will load the `DevToolsSettingsModalLazy` component lazily with - * a predefined fallback and error boundary. - */ -export const DevToolsSettingsModal = withSuspense(DevToolsSettingsModalLazy); - -/** - * The Lazily-loaded `DevToolsVariablesFlyout` component. Consumers should use `React.Suspense` or - * the withSuspense` HOC to load this component. - */ -export const DevToolsVariablesFlyoutLazy = React.lazy(() => - import('./variables').then(({ DevToolsVariablesFlyout }) => ({ - default: DevToolsVariablesFlyout, - })) -); - -/** - * A `DevToolsVariablesFlyout` component that is wrapped by the `withSuspense` HOC. This component can - * be used directly by consumers and will load the `DevToolsVariablesFlyoutLazy` component lazily with - * a predefined fallback and error boundary. - */ -export const DevToolsVariablesFlyout = withSuspense(DevToolsVariablesFlyoutLazy); +export { OutputPanelEmptyState } from './output_panel_empty_state'; +export { HelpPopover } from './help_popover'; +export { ShortcutsPopover } from './shortcuts_popover'; +export type { DevToolsVariable } from './variables/types'; +export { ConsoleTourStep, type ConsoleTourStepProps } from './console_tour_step'; diff --git a/src/plugins/console/public/application/components/output_panel_empty_state.tsx b/src/plugins/console/public/application/components/output_panel_empty_state.tsx new file mode 100644 index 0000000000000..6fdda1b5e3c5f --- /dev/null +++ b/src/plugins/console/public/application/components/output_panel_empty_state.tsx @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import React, { FunctionComponent } from 'react'; +import { EuiEmptyPrompt, EuiTitle, EuiLink } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { useServicesContext } from '../contexts'; + +export const OutputPanelEmptyState: FunctionComponent = () => { + const { docLinks } = useServicesContext(); + + return ( + + + + } + body={ +

+ +

+ } + footer={ + <> + +

+ +

+ + + + + + } + data-test-subj="consoleOutputPanelEmptyState" + /> + ); +}; diff --git a/src/plugins/console/public/application/components/settings/index.ts b/src/plugins/console/public/application/components/settings/index.ts new file mode 100644 index 0000000000000..b446307a04a01 --- /dev/null +++ b/src/plugins/console/public/application/components/settings/index.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import React from 'react'; +import { withSuspense } from '@kbn/shared-ux-utility'; + +export { type Props } from './settings_editor'; +export { type AutocompleteOptions } from './types'; + +/** + * The Lazily-loaded `SettingsEditorLazy` component. Consumers should use `React.Suspense` or + * the withSuspense` HOC to load this component. + */ +export const SettingsEditorLazy = React.lazy(() => + import('./settings_editor').then(({ SettingsEditor }) => ({ + default: SettingsEditor, + })) +); + +/** + * A `SettingsEditor` component that is wrapped by the `withSuspense` HOC. This component can + * be used directly by consumers and will load the `SettingsEditorLazy` component lazily with + * a predefined fallback and error boundary. + */ +export const SettingsEditor = withSuspense(SettingsEditorLazy); diff --git a/src/plugins/console/public/application/components/settings/settings_editor.tsx b/src/plugins/console/public/application/components/settings/settings_editor.tsx new file mode 100644 index 0000000000000..6f2bef834a559 --- /dev/null +++ b/src/plugins/console/public/application/components/settings/settings_editor.tsx @@ -0,0 +1,371 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { debounce } from 'lodash'; +import React, { useState, useCallback, useEffect, useRef } from 'react'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; + +import { + EuiButton, + EuiFieldNumber, + EuiSwitch, + EuiSuperSelect, + EuiTitle, + EuiSpacer, + EuiText, +} from '@elastic/eui'; + +import { SettingsGroup } from './settings_group'; +import { SettingsFormRow } from './settings_form_row'; +import { DevToolsSettings } from '../../../services'; + +const DEBOUNCE_DELAY = 500; +const ON_LABEL = i18n.translate('console.settingsPage.onLabel', { defaultMessage: 'On' }); +const OFF_LABEL = i18n.translate('console.settingsPage.offLabel', { defaultMessage: 'Off' }); + +const onceTimeInterval = () => + i18n.translate('console.settingsPage.refreshInterval.onceTimeInterval', { + defaultMessage: 'Once, when console loads', + }); + +const everyNMinutesTimeInterval = (value: number) => + i18n.translate('console.settingsPage.refreshInterval.everyNMinutesTimeInterval', { + defaultMessage: 'Every {value} {value, plural, one {minute} other {minutes}}', + values: { value }, + }); + +const everyHourTimeInterval = () => + i18n.translate('console.settingsPage.refreshInterval.everyHourTimeInterval', { + defaultMessage: 'Every hour', + }); + +const PRESETS_IN_MINUTES = [0, 1, 10, 20, 60]; +const intervalOptions = PRESETS_IN_MINUTES.map((value) => ({ + value: (value * 60000).toString(), + inputDisplay: + value === 0 + ? onceTimeInterval() + : value === 60 + ? everyHourTimeInterval() + : everyNMinutesTimeInterval(value), +})); + +export interface Props { + onSaveSettings: (newSettings: DevToolsSettings) => void; + refreshAutocompleteSettings: (selectedSettings: DevToolsSettings['autocomplete']) => void; + settings: DevToolsSettings; +} + +export const SettingsEditor = (props: Props) => { + const isMounted = useRef(false); + + const [fontSize, setFontSize] = useState(props.settings.fontSize); + const [wrapMode, setWrapMode] = useState(props.settings.wrapMode); + const [fields, setFields] = useState(props.settings.autocomplete.fields); + const [indices, setIndices] = useState(props.settings.autocomplete.indices); + const [templates, setTemplates] = useState(props.settings.autocomplete.templates); + const [dataStreams, setDataStreams] = useState(props.settings.autocomplete.dataStreams); + const [polling, setPolling] = useState(props.settings.polling); + const [pollInterval, setPollInterval] = useState(props.settings.pollInterval); + const [tripleQuotes, setTripleQuotes] = useState(props.settings.tripleQuotes); + const [isHistoryEnabled, setIsHistoryEnabled] = useState(props.settings.isHistoryEnabled); + const [isKeyboardShortcutsEnabled, setIsKeyboardShortcutsEnabled] = useState( + props.settings.isKeyboardShortcutsEnabled + ); + const [isAccessibilityOverlayEnabled, setIsAccessibilityOverlayEnabled] = useState( + props.settings.isAccessibilityOverlayEnabled + ); + + const autoCompleteCheckboxes = [ + { + id: 'fields', + label: i18n.translate('console.settingsPage.fieldsLabelText', { + defaultMessage: 'Fields', + }), + stateSetter: setFields, + checked: fields, + }, + { + id: 'indices', + label: i18n.translate('console.settingsPage.indicesAndAliasesLabelText', { + defaultMessage: 'Indices and aliases', + }), + stateSetter: setIndices, + checked: indices, + }, + { + id: 'templates', + label: i18n.translate('console.settingsPage.templatesLabelText', { + defaultMessage: 'Templates', + }), + stateSetter: setTemplates, + checked: templates, + }, + { + id: 'dataStreams', + label: i18n.translate('console.settingsPage.dataStreamsLabelText', { + defaultMessage: 'Data streams', + }), + stateSetter: setDataStreams, + checked: dataStreams, + }, + ]; + + const saveSettings = () => { + props.onSaveSettings({ + fontSize, + wrapMode, + autocomplete: { + fields, + indices, + templates, + dataStreams, + }, + polling, + pollInterval, + tripleQuotes, + isHistoryEnabled, + isKeyboardShortcutsEnabled, + isAccessibilityOverlayEnabled, + }); + }; + const debouncedSaveSettings = debounce(saveSettings, DEBOUNCE_DELAY); + + useEffect(() => { + if (isMounted.current) { + debouncedSaveSettings(); + } else { + isMounted.current = true; + } + }, [ + fontSize, + wrapMode, + fields, + indices, + templates, + dataStreams, + polling, + pollInterval, + tripleQuotes, + isHistoryEnabled, + isKeyboardShortcutsEnabled, + isAccessibilityOverlayEnabled, + debouncedSaveSettings, + ]); + + const onPollingIntervalChange = useCallback((value: string) => { + const sanitizedValue = parseInt(value, 10); + + setPolling(!!sanitizedValue); + setPollInterval(sanitizedValue); + }, []); + + const toggleKeyboardShortcuts = useCallback((isEnabled: boolean) => { + setIsKeyboardShortcutsEnabled(isEnabled); + }, []); + + const toggleAccessibilityOverlay = useCallback( + (isEnabled: boolean) => setIsAccessibilityOverlayEnabled(isEnabled), + [] + ); + + const toggleSavingToHistory = useCallback( + (isEnabled: boolean) => setIsHistoryEnabled(isEnabled), + [] + ); + + return ( + <> + +

+ +

+ + + +

+ +

+ + + {/* GENERAL SETTINGS */} + + + toggleSavingToHistory(e.target.checked)} + /> + + + toggleKeyboardShortcuts(e.target.checked)} + /> + + + toggleAccessibilityOverlay(e.target.checked)} + /> + + + {/* DISPLAY SETTINGS */} + + + { + const val = parseInt(e.target.value, 10); + if (!val) return; + setFontSize(val); + }} + /> + + + setWrapMode(e.target.checked)} + id="wrapLines" + /> + + + setTripleQuotes(e.target.checked)} + id="tripleQuotes" + /> + + + {/* AUTOCOMPLETE SETTINGS */} + + {autoCompleteCheckboxes.map((opts) => ( + + opts.stateSetter(e.target.checked)} + /> + + ))} + + {/* AUTOCOMPLETE REFRESH SETTINGS */} + {(fields || indices || templates || dataStreams) && ( + <> + + + + + + + { + // Only refresh the currently selected settings. + props.refreshAutocompleteSettings({ + fields, + indices, + templates, + dataStreams, + }); + }} + > + + + + + )} + + ); +}; diff --git a/src/plugins/console/public/application/components/settings/settings_form_row.tsx b/src/plugins/console/public/application/components/settings/settings_form_row.tsx new file mode 100644 index 0000000000000..383eabfb93bd2 --- /dev/null +++ b/src/plugins/console/public/application/components/settings/settings_form_row.tsx @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import React from 'react'; + +import { EuiFlexGroup, EuiFlexItem, EuiFormRow } from '@elastic/eui'; + +export interface DevToolsSettingsModalProps { + label: string; + children: React.ReactNode; +} + +export const SettingsFormRow = ({ label, children }: DevToolsSettingsModalProps) => { + return ( + + + + + {label} + + + + {children} + + + ); +}; diff --git a/src/plugins/console/public/application/components/settings/settings_group.tsx b/src/plugins/console/public/application/components/settings/settings_group.tsx new file mode 100644 index 0000000000000..d6feb8af1c90a --- /dev/null +++ b/src/plugins/console/public/application/components/settings/settings_group.tsx @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import React from 'react'; + +import { EuiTitle, EuiSpacer, EuiText, EuiHorizontalRule } from '@elastic/eui'; + +export interface DevToolsSettingsModalProps { + title: string; + description?: string; +} + +export const SettingsGroup = ({ title, description }: DevToolsSettingsModalProps) => { + return ( + <> + + +

{title}

+ + {description && ( + <> + + +

{description}

+ + + )} + + + ); +}; diff --git a/src/plugins/console/public/application/containers/split_panel/index.ts b/src/plugins/console/public/application/components/settings/types.ts similarity index 84% rename from src/plugins/console/public/application/containers/split_panel/index.ts rename to src/plugins/console/public/application/components/settings/types.ts index 00aa2b83db6d8..f524e37124746 100644 --- a/src/plugins/console/public/application/containers/split_panel/index.ts +++ b/src/plugins/console/public/application/components/settings/types.ts @@ -7,5 +7,4 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -export { Panel } from './panel'; -export { PanelsContainer } from './panel_container'; +export type AutocompleteOptions = 'fields' | 'indices' | 'templates'; diff --git a/src/plugins/console/public/application/components/settings_modal.tsx b/src/plugins/console/public/application/components/settings_modal.tsx deleted file mode 100644 index 9b7740b4affdf..0000000000000 --- a/src/plugins/console/public/application/components/settings_modal.tsx +++ /dev/null @@ -1,391 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import _ from 'lodash'; -import React, { Fragment, useState, useCallback } from 'react'; -import { i18n } from '@kbn/i18n'; -import { FormattedMessage } from '@kbn/i18n-react'; - -import { - EuiButton, - EuiButtonEmpty, - EuiFieldNumber, - EuiFormRow, - EuiCheckboxGroup, - EuiModal, - EuiModalBody, - EuiModalFooter, - EuiModalHeader, - EuiModalHeaderTitle, - EuiSwitch, - EuiSuperSelect, -} from '@elastic/eui'; - -import { DevToolsSettings } from '../../services'; -import { unregisterCommands } from '../containers/editor/legacy/console_editor/keyboard_shortcuts'; -import type { SenseEditor } from '../models'; - -export type AutocompleteOptions = 'fields' | 'indices' | 'templates'; - -const onceTimeInterval = () => - i18n.translate('console.settingsPage.refreshInterval.onceTimeInterval', { - defaultMessage: 'Once, when console loads', - }); - -const everyNMinutesTimeInterval = (value: number) => - i18n.translate('console.settingsPage.refreshInterval.everyNMinutesTimeInterval', { - defaultMessage: 'Every {value} {value, plural, one {minute} other {minutes}}', - values: { value }, - }); - -const everyHourTimeInterval = () => - i18n.translate('console.settingsPage.refreshInterval.everyHourTimeInterval', { - defaultMessage: 'Every hour', - }); - -const PRESETS_IN_MINUTES = [0, 1, 10, 20, 60]; -const intervalOptions = PRESETS_IN_MINUTES.map((value) => ({ - value: (value * 60000).toString(), - inputDisplay: - value === 0 - ? onceTimeInterval() - : value === 60 - ? everyHourTimeInterval() - : everyNMinutesTimeInterval(value), -})); - -export interface DevToolsSettingsModalProps { - onSaveSettings: (newSettings: DevToolsSettings) => void; - onClose: () => void; - refreshAutocompleteSettings: (selectedSettings: DevToolsSettings['autocomplete']) => void; - settings: DevToolsSettings; - editorInstance: SenseEditor | null; -} - -export const DevToolsSettingsModal = (props: DevToolsSettingsModalProps) => { - const [fontSize, setFontSize] = useState(props.settings.fontSize); - const [wrapMode, setWrapMode] = useState(props.settings.wrapMode); - const [fields, setFields] = useState(props.settings.autocomplete.fields); - const [indices, setIndices] = useState(props.settings.autocomplete.indices); - const [templates, setTemplates] = useState(props.settings.autocomplete.templates); - const [dataStreams, setDataStreams] = useState(props.settings.autocomplete.dataStreams); - const [polling, setPolling] = useState(props.settings.polling); - const [pollInterval, setPollInterval] = useState(props.settings.pollInterval); - const [tripleQuotes, setTripleQuotes] = useState(props.settings.tripleQuotes); - const [isHistoryEnabled, setIsHistoryEnabled] = useState(props.settings.isHistoryEnabled); - const [isKeyboardShortcutsEnabled, setIsKeyboardShortcutsEnabled] = useState( - props.settings.isKeyboardShortcutsEnabled - ); - const [isAccessibilityOverlayEnabled, setIsAccessibilityOverlayEnabled] = useState( - props.settings.isAccessibilityOverlayEnabled - ); - - const autoCompleteCheckboxes = [ - { - id: 'fields', - label: i18n.translate('console.settingsPage.fieldsLabelText', { - defaultMessage: 'Fields', - }), - stateSetter: setFields, - }, - { - id: 'indices', - label: i18n.translate('console.settingsPage.indicesAndAliasesLabelText', { - defaultMessage: 'Indices and aliases', - }), - stateSetter: setIndices, - }, - { - id: 'templates', - label: i18n.translate('console.settingsPage.templatesLabelText', { - defaultMessage: 'Templates', - }), - stateSetter: setTemplates, - }, - { - id: 'dataStreams', - label: i18n.translate('console.settingsPage.dataStreamsLabelText', { - defaultMessage: 'Data streams', - }), - stateSetter: setDataStreams, - }, - ]; - - const checkboxIdToSelectedMap = { - fields, - indices, - templates, - dataStreams, - }; - - const onAutocompleteChange = (optionId: AutocompleteOptions) => { - const option = _.find(autoCompleteCheckboxes, (item) => item.id === optionId); - if (option) { - option.stateSetter(!checkboxIdToSelectedMap[optionId]); - } - }; - - function saveSettings() { - props.onSaveSettings({ - fontSize, - wrapMode, - autocomplete: { - fields, - indices, - templates, - dataStreams, - }, - polling, - pollInterval, - tripleQuotes, - isHistoryEnabled, - isKeyboardShortcutsEnabled, - isAccessibilityOverlayEnabled, - }); - } - - const onPollingIntervalChange = useCallback((value: string) => { - const sanitizedValue = parseInt(value, 10); - - setPolling(!!sanitizedValue); - setPollInterval(sanitizedValue); - }, []); - - const toggleKeyboardShortcuts = useCallback( - (isEnabled: boolean) => { - if (props.editorInstance) { - unregisterCommands(props.editorInstance); - } - - setIsKeyboardShortcutsEnabled(isEnabled); - }, - [props.editorInstance] - ); - - const toggleAccessibilityOverlay = useCallback( - (isEnabled: boolean) => setIsAccessibilityOverlayEnabled(isEnabled), - [] - ); - - const toggleSavingToHistory = useCallback( - (isEnabled: boolean) => setIsHistoryEnabled(isEnabled), - [] - ); - - // It only makes sense to show polling options if the user needs to fetch any data. - const pollingFields = - fields || indices || templates || dataStreams ? ( - - - } - helpText={ - - } - > - - - - { - // Only refresh the currently selected settings. - props.refreshAutocompleteSettings({ - fields, - indices, - templates, - dataStreams, - }); - }} - > - - - - ) : undefined; - - return ( - - - - - - - - - - } - > - { - const val = parseInt(e.target.value, 10); - if (!val) return; - setFontSize(val); - }} - /> - - - - - } - onChange={(e) => setWrapMode(e.target.checked)} - /> - - - - } - > - - } - onChange={(e) => setTripleQuotes(e.target.checked)} - /> - - - - } - > - - } - onChange={(e) => toggleSavingToHistory(e.target.checked)} - /> - - - - } - > - - } - onChange={(e) => toggleKeyboardShortcuts(e.target.checked)} - /> - - - - } - > - - } - onChange={(e) => toggleAccessibilityOverlay(e.target.checked)} - /> - - - - } - > - { - const { stateSetter, ...rest } = opts; - return rest; - })} - idToSelectedMap={checkboxIdToSelectedMap} - onChange={(e: unknown) => { - onAutocompleteChange(e as AutocompleteOptions); - }} - /> - - - {pollingFields} - - - - - - - - - - - - - ); -}; diff --git a/src/plugins/console/public/application/containers/editor/legacy/console_editor/index.ts b/src/plugins/console/public/application/components/shortcuts_popover/index.ts similarity index 84% rename from src/plugins/console/public/application/containers/editor/legacy/console_editor/index.ts rename to src/plugins/console/public/application/components/shortcuts_popover/index.ts index 3df103aa3be70..67b91a4e6dffb 100644 --- a/src/plugins/console/public/application/containers/editor/legacy/console_editor/index.ts +++ b/src/plugins/console/public/application/components/shortcuts_popover/index.ts @@ -7,5 +7,4 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -export { Editor } from './editor'; -export { EditorOutput } from './editor_output'; +export { ShortcutsPopover } from './shortcuts_popover'; diff --git a/src/plugins/console/public/application/components/shortcuts_popover/keys.tsx b/src/plugins/console/public/application/components/shortcuts_popover/keys.tsx new file mode 100644 index 0000000000000..9a4a0329cbf5c --- /dev/null +++ b/src/plugins/console/public/application/components/shortcuts_popover/keys.tsx @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import React from 'react'; +import { i18n } from '@kbn/i18n'; +import { EuiIcon } from '@elastic/eui'; + +export const KEYS = { + keyCtrlCmd: i18n.translate('console.shortcutKeys.keyCtrlCmd', { + defaultMessage: 'Ctrl/Cmd', + }), + keyEnter: i18n.translate('console.shortcutKeys.keyEnter', { + defaultMessage: 'Enter', + }), + keyAltOption: i18n.translate('console.shortcutKeys.keyAltOption', { + defaultMessage: 'Alt/Option', + }), + keyOption: i18n.translate('console.shortcutKeys.keyOption', { + defaultMessage: 'Option', + }), + keyShift: i18n.translate('console.shortcutKeys.keyShift', { + defaultMessage: 'Shift', + }), + keyTab: i18n.translate('console.shortcutKeys.keyTab', { + defaultMessage: 'Tab', + }), + keyEsc: i18n.translate('console.shortcutKeys.keyEsc', { + defaultMessage: 'Esc', + }), + keyUp: ( + + ), + keyDown: ( + + ), + keySlash: i18n.translate('console.shortcutKeys.keySlash', { + defaultMessage: '/', + }), + keySpace: i18n.translate('console.shortcutKeys.keySpace', { + defaultMessage: 'Space', + }), + keyI: i18n.translate('console.shortcutKeys.keyI', { + defaultMessage: 'I', + }), + keyO: i18n.translate('console.shortcutKeys.keyO', { + defaultMessage: 'O', + }), + keyL: i18n.translate('console.shortcutKeys.keyL', { + defaultMessage: 'L', + }), +}; diff --git a/src/plugins/console/public/application/components/shortcuts_popover/shortcut_line.tsx b/src/plugins/console/public/application/components/shortcuts_popover/shortcut_line.tsx new file mode 100644 index 0000000000000..a57c256ce3ce5 --- /dev/null +++ b/src/plugins/console/public/application/components/shortcuts_popover/shortcut_line.tsx @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { EuiCode, EuiFlexGroup, EuiFlexItem, EuiText } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import React from 'react'; + +interface ShortcutLineFlexItemProps { + id: string; + description: string; + keys: any[]; + alternativeKeys?: any[]; +} + +const renderKeys = (keys: string[]) => { + return keys.map((key, index) => ( + + {index > 0 && ' + '} + {key} + + )); +}; + +export const ShortcutLineFlexItem = ({ + id, + description, + keys, + alternativeKeys, +}: ShortcutLineFlexItemProps) => { + return ( + + + + + {i18n.translate('console.shortcutDescription.' + id, { + defaultMessage: description, + })} + + + + + {renderKeys(keys)} + {alternativeKeys && ( + <> + + {' '} + {i18n.translate('console.shortcuts.alternativeKeysOrDivider', { + defaultMessage: 'or', + })}{' '} + + {renderKeys(alternativeKeys)} + + )} + + + + + ); +}; diff --git a/src/plugins/console/public/application/components/shortcuts_popover/shortcuts_popover.tsx b/src/plugins/console/public/application/components/shortcuts_popover/shortcuts_popover.tsx new file mode 100644 index 0000000000000..9ceaf13dc5dba --- /dev/null +++ b/src/plugins/console/public/application/components/shortcuts_popover/shortcuts_popover.tsx @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import React from 'react'; +import { EuiPopover, EuiTitle, EuiHorizontalRule, EuiFlexGroup, EuiSpacer } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { ShortcutLineFlexItem } from './shortcut_line'; +import { KEYS } from './keys'; + +interface ShortcutsPopoverProps { + button: any; + isOpen: boolean; + closePopover: () => void; +} + +export const ShortcutsPopover = ({ button, isOpen, closePopover }: ShortcutsPopoverProps) => { + return ( + + +
+ {i18n.translate('console.shortcuts.navigationShortcutsSubtitle', { + defaultMessage: 'Navigation shortcuts', + })} +
+ + + + + + + + +
+ {i18n.translate('console.shortcuts.requestShortcutsSubtitle', { + defaultMessage: 'Request shortcuts', + })} +
+ + + + + + + + + + + + +
+ {i18n.translate('console.shortcuts.autocompleteShortcutsSubtitle', { + defaultMessage: 'Autocomplete menu shortcuts', + })} +
+ + + + + + + + + ); +}; diff --git a/src/plugins/console/public/application/components/top_nav_menu.tsx b/src/plugins/console/public/application/components/top_nav_menu.tsx index cddbe95f8f1b5..2309c01afc18a 100644 --- a/src/plugins/console/public/application/components/top_nav_menu.tsx +++ b/src/plugins/console/public/application/components/top_nav_menu.tsx @@ -9,6 +9,7 @@ import React, { FunctionComponent } from 'react'; import { EuiTabs, EuiTab } from '@elastic/eui'; +import { ConsoleTourStep, ConsoleTourStepProps } from './console_tour_step'; export interface TopNavMenuItem { id: string; @@ -16,28 +17,42 @@ export interface TopNavMenuItem { description: string; onClick: () => void; testId: string; + isSelected: boolean; + tourStep?: number; } interface Props { disabled?: boolean; items: TopNavMenuItem[]; + tourStepProps: ConsoleTourStepProps[]; } -export const TopNavMenu: FunctionComponent = ({ items, disabled }) => { +export const TopNavMenu: FunctionComponent = ({ items, disabled, tourStepProps }) => { return ( - + {items.map((item, idx) => { - return ( + const tab = ( {item.label} ); + + if (item.tourStep) { + return ( + + {tab} + + ); + } + + return tab; })} ); diff --git a/src/plugins/console/public/application/components/variables/index.ts b/src/plugins/console/public/application/components/variables/index.ts index 108befce39f51..8051ed2ddaa93 100644 --- a/src/plugins/console/public/application/components/variables/index.ts +++ b/src/plugins/console/public/application/components/variables/index.ts @@ -7,5 +7,25 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -export * from './variables_flyout'; -export * from './utils'; +import React from 'react'; +import { withSuspense } from '@kbn/shared-ux-utility'; + +export { type Props } from './variables_editor'; +export { type DevToolsVariable } from './types'; + +/** + * The Lazily-loaded `VariablesEditorLazy` component. Consumers should use `React.Suspense` or + * the withSuspense` HOC to load this component. + */ +export const VariablesEditorLazy = React.lazy(() => + import('./variables_editor').then(({ VariablesEditor }) => ({ + default: VariablesEditor, + })) +); + +/** + * A `VariablesEditor` component that is wrapped by the `withSuspense` HOC. This component can + * be used directly by consumers and will load the `VariablesEditorLazy` component lazily with + * a predefined fallback and error boundary. + */ +export const VariablesEditor = withSuspense(VariablesEditorLazy); diff --git a/src/plugins/console/public/application/components/variables/types.ts b/src/plugins/console/public/application/components/variables/types.ts new file mode 100644 index 0000000000000..40a2ac86c361f --- /dev/null +++ b/src/plugins/console/public/application/components/variables/types.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +export interface DevToolsVariable { + id: string; + name: string; + value: string; +} diff --git a/src/plugins/console/public/application/components/variables/utils.ts b/src/plugins/console/public/application/components/variables/utils.ts index 50664e0a99cf6..b636b9b0a6266 100644 --- a/src/plugins/console/public/application/components/variables/utils.ts +++ b/src/plugins/console/public/application/components/variables/utils.ts @@ -7,38 +7,18 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import { v4 as uuidv4 } from 'uuid'; -import type { DevToolsVariable } from './variables_flyout'; +import { type DevToolsVariable } from './types'; -export const editVariable = ( - name: string, - value: string, - id: string, - variables: DevToolsVariable[] -) => { - const index = variables.findIndex((v) => v.id === id); - - if (index === -1) { - return variables; - } - - return [ - ...variables.slice(0, index), - { ...variables[index], [name]: value }, - ...variables.slice(index + 1), - ]; +export const editVariable = (newVariable: DevToolsVariable, variables: DevToolsVariable[]) => { + return variables.map((variable: DevToolsVariable) => { + return variable.id === newVariable.id ? newVariable : variable; + }); }; export const deleteVariable = (variables: DevToolsVariable[], id: string) => { return variables.filter((v) => v.id !== id); }; -export const generateEmptyVariableField = (): DevToolsVariable => ({ - id: uuidv4(), - name: '', - value: '', -}); - export const isValidVariableName = (name: string) => { /* * MUST avoid characters that get URL-encoded, because they'll result in unusable variable names. diff --git a/src/plugins/console/public/application/components/variables/variables_editor.tsx b/src/plugins/console/public/application/components/variables/variables_editor.tsx new file mode 100644 index 0000000000000..197fdec7f49c7 --- /dev/null +++ b/src/plugins/console/public/application/components/variables/variables_editor.tsx @@ -0,0 +1,255 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import React, { useState, useCallback, useEffect, useRef } from 'react'; +import { BehaviorSubject } from 'rxjs'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; + +import { + EuiTitle, + EuiButton, + EuiBasicTable, + EuiButtonIcon, + EuiSpacer, + EuiText, + EuiCode, + useGeneratedHtmlId, + EuiConfirmModal, + type EuiBasicTableColumn, +} from '@elastic/eui'; + +import { VariableEditorForm } from './variables_editor_form'; +import * as utils from './utils'; +import { type DevToolsVariable } from './types'; + +export interface Props { + onSaveVariables: (newVariables: DevToolsVariable[]) => void; + variables: []; +} + +export const VariablesEditor = (props: Props) => { + const isMounted = useRef(false); + const [isAddingVariable, setIsAddingVariable] = useState(false); + const [deleteModalForVariable, setDeleteModalForVariable] = useState(null); + const [variables, setVariables] = useState(props.variables); + const deleteModalTitleId = useGeneratedHtmlId(); + + // Use a ref to persist the BehaviorSubject across renders + const itemIdToExpandedRowMap$ = useRef(new BehaviorSubject>({})); + // Subscribe to the BehaviorSubject and update local state on change + const [itemIdToExpandedRowMap, setItemIdToExpandedRowMap] = useState< + Record + >({}); + // Clear the expanded row map and dispose all the expanded rows + const collapseExpandedRows = () => itemIdToExpandedRowMap$.current.next({}); + + // Subscribe to the BehaviorSubject on mount + useEffect(() => { + const subscription = itemIdToExpandedRowMap$.current.subscribe(setItemIdToExpandedRowMap); + return () => subscription.unsubscribe(); + }, []); + + // Always save variables when they change + useEffect(() => { + if (isMounted.current) { + props.onSaveVariables(variables); + } else { + isMounted.current = true; + } + }, [variables, props]); + + const toggleDetails = (variableId: string) => { + const currentMap = itemIdToExpandedRowMap$.current.getValue(); + let itemIdToExpandedRowMapValues = { ...currentMap }; + + if (itemIdToExpandedRowMapValues[variableId]) { + delete itemIdToExpandedRowMapValues[variableId]; + } else { + // Always close the add variable form when editing a variable + setIsAddingVariable(false); + // We only allow one expanded row at a time + itemIdToExpandedRowMapValues = {}; + itemIdToExpandedRowMapValues[variableId] = ( + { + const updatedVariables = utils.editVariable(data, variables); + setVariables(updatedVariables); + collapseExpandedRows(); + }} + onCancel={() => { + collapseExpandedRows(); + }} + defaultValue={variables.find((v) => v.id === variableId)} + /> + ); + } + + // Update the BehaviorSubject with the new state + itemIdToExpandedRowMap$.current.next(itemIdToExpandedRowMapValues); + }; + + const deleteVariable = useCallback( + (id: string) => { + const updatedVariables = utils.deleteVariable(variables, id); + setVariables(updatedVariables); + setDeleteModalForVariable(null); + }, + [variables, setDeleteModalForVariable] + ); + + const onAddVariable = (data: DevToolsVariable) => { + setVariables((v: DevToolsVariable[]) => [...v, data]); + setIsAddingVariable(false); + }; + + const columns: Array> = [ + { + field: 'name', + name: i18n.translate('console.variablesPage.variablesTable.columns.variableHeader', { + defaultMessage: 'Variable name', + }), + 'data-test-subj': 'variableNameCell', + render: (name: string) => { + return {`\$\{${name}\}`}; + }, + }, + { + field: 'value', + name: i18n.translate('console.variablesPage.variablesTable.columns.valueHeader', { + defaultMessage: 'Value', + }), + 'data-test-subj': 'variableValueCell', + render: (value: string) => {value}, + }, + { + field: 'id', + name: '', + width: '40px', + isExpander: true, + render: (id: string, variable: DevToolsVariable) => { + const itemIdToExpandedRowMapValues = { ...itemIdToExpandedRowMap }; + + return ( + toggleDetails(id)} + data-test-subj="variableEditButton" + /> + ); + }, + }, + { + field: 'id', + name: '', + width: '40px', + render: (id: string, variable: DevToolsVariable) => ( + setDeleteModalForVariable(id)} + data-test-subj="variablesRemoveButton" + /> + ), + }, + ]; + + return ( + <> + +

+ +

+ + + +

+ +

+ + + + + + {isAddingVariable && ( + setIsAddingVariable(false)} /> + )} + + + +
+ { + setIsAddingVariable(true); + collapseExpandedRows(); + }} + disabled={isAddingVariable} + > + + +
+ + {deleteModalForVariable && ( + setDeleteModalForVariable(null)} + onConfirm={() => deleteVariable(deleteModalForVariable)} + cancelButtonText={i18n.translate('console.variablesPage.deleteModal.cancelButtonText', { + defaultMessage: 'Cancel', + })} + confirmButtonText={i18n.translate('console.variablesPage.deleteModal.confirmButtonText', { + defaultMessage: 'Delete variable', + })} + buttonColor="danger" + > +

+ +

+ + )} + + ); +}; diff --git a/src/plugins/console/public/application/components/variables/variables_editor_form.tsx b/src/plugins/console/public/application/components/variables/variables_editor_form.tsx new file mode 100644 index 0000000000000..446aaab0d4e94 --- /dev/null +++ b/src/plugins/console/public/application/components/variables/variables_editor_form.tsx @@ -0,0 +1,184 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import React from 'react'; +import { i18n } from '@kbn/i18n'; +import { v4 as uuidv4 } from 'uuid'; +import { FormattedMessage } from '@kbn/i18n-react'; + +import { + EuiTitle, + EuiFlexGroup, + EuiFlexItem, + EuiButton, + EuiSpacer, + EuiPanel, + EuiButtonEmpty, +} from '@elastic/eui'; + +import { + useForm, + Form, + UseField, + TextField, + FieldConfig, + fieldValidators, + FormConfig, + ValidationFuncArg, +} from '../../../shared_imports'; + +import { type DevToolsVariable } from './types'; +import { isValidVariableName } from './utils'; + +export interface VariableEditorFormProps { + onSubmit: (data: DevToolsVariable) => void; + onCancel: () => void; + defaultValue?: DevToolsVariable; + title?: string; +} + +const fieldsConfig: Record = { + variableName: { + label: i18n.translate('console.variablesPage.form.variableNameFieldLabel', { + defaultMessage: 'Variable name', + }), + validations: [ + { + validator: ({ value }: ValidationFuncArg) => { + if (value.trim() === '') { + return { + message: i18n.translate('console.variablesPage.form.variableNameRequiredLabel', { + defaultMessage: 'This is a required field', + }), + }; + } + + if (!isValidVariableName(value)) { + return { + message: i18n.translate('console.variablesPage.form.variableNameInvalidLabel', { + defaultMessage: 'Only letters, numbers and underscores are allowed', + }), + }; + } + }, + }, + ], + }, + value: { + label: i18n.translate('console.variablesPage.form.valueFieldLabel', { + defaultMessage: 'Value', + }), + validations: [ + { + validator: fieldValidators.emptyField( + i18n.translate('console.variablesPage.form.valueRequiredLabel', { + defaultMessage: 'Value is required', + }) + ), + }, + ], + }, +}; + +export const VariableEditorForm = (props: VariableEditorFormProps) => { + const onSubmit: FormConfig['onSubmit'] = async (data, isValid) => { + if (isValid) { + props.onSubmit({ + ...props.defaultValue, + ...data, + ...(props.defaultValue ? {} : { id: uuidv4() }), + } as DevToolsVariable); + } + }; + + const { form } = useForm({ onSubmit, defaultValue: props.defaultValue }); + + return ( + <> + + +

+ {props.title ?? ( + + )} +

+ + + +
+ + + + + + + + + props.onCancel()}> + + + + + + + + + + + +
+ + ); +}; diff --git a/src/plugins/console/public/application/components/variables/variables_flyout.tsx b/src/plugins/console/public/application/components/variables/variables_flyout.tsx deleted file mode 100644 index 9211d2a7e524f..0000000000000 --- a/src/plugins/console/public/application/components/variables/variables_flyout.tsx +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import React, { useState, useCallback, ChangeEvent, FormEvent } from 'react'; -import { i18n } from '@kbn/i18n'; -import { FormattedMessage } from '@kbn/i18n-react'; - -import { - EuiFlyout, - EuiFlyoutHeader, - EuiTitle, - EuiFlyoutBody, - EuiFlyoutFooter, - EuiFlexGroup, - EuiFlexItem, - EuiButton, - EuiButtonEmpty, - EuiBasicTable, - EuiFieldText, - useGeneratedHtmlId, - EuiForm, - EuiFormRow, - EuiButtonIcon, - EuiSpacer, - EuiText, - type EuiBasicTableColumn, -} from '@elastic/eui'; - -import * as utils from './utils'; - -export interface DevToolsVariablesFlyoutProps { - onClose: () => void; - onSaveVariables: (newVariables: DevToolsVariable[]) => void; - variables: []; -} - -export interface DevToolsVariable { - id: string; - name: string; - value: string; -} - -export const DevToolsVariablesFlyout = (props: DevToolsVariablesFlyoutProps) => { - const [variables, setVariables] = useState(props.variables); - const formId = useGeneratedHtmlId({ prefix: '__console' }); - - const addNewVariable = useCallback(() => { - setVariables((v) => [...v, utils.generateEmptyVariableField()]); - }, []); - - const deleteVariable = useCallback( - (id: string) => { - const updatedVariables = utils.deleteVariable(variables, id); - setVariables(updatedVariables); - }, - [variables] - ); - - const onSubmit = useCallback( - (e: FormEvent) => { - e.preventDefault(); - props.onSaveVariables(variables.filter(({ name, value }) => name.trim() && value)); - }, - [props, variables] - ); - - const onChange = useCallback( - (event: ChangeEvent, id: string) => { - const { name, value } = event.target; - const editedVariables = utils.editVariable(name, value, id, variables); - setVariables(editedVariables); - }, - [variables] - ); - - const columns: Array> = [ - { - field: 'name', - name: i18n.translate('console.variablesPage.variablesTable.columns.variableHeader', { - defaultMessage: 'Variable name', - }), - render: (name, { id }) => { - const isInvalid = !utils.isValidVariableName(name); - return ( - , - ]} - fullWidth={true} - css={{ flexGrow: 1 }} - > - onChange(e, id)} - isInvalid={isInvalid} - fullWidth={true} - aria-label={i18n.translate( - 'console.variablesPage.variablesTable.variableInput.ariaLabel', - { - defaultMessage: 'Variable name', - } - )} - /> - - ); - }, - }, - { - field: 'value', - name: i18n.translate('console.variablesPage.variablesTable.columns.valueHeader', { - defaultMessage: 'Value', - }), - render: (value, { id }) => ( - onChange(e, id)} - value={value} - aria-label={i18n.translate('console.variablesPage.variablesTable.valueInput.ariaLabel', { - defaultMessage: 'Variable value', - })} - /> - ), - }, - { - field: 'id', - name: '', - width: '5%', - render: (id: string) => ( - deleteVariable(id)} - data-test-subj="variablesRemoveButton" - /> - ), - }, - ]; - - return ( - - - -

- -

- - - -

- - - - ), - }} - /> -

- - - - - - - - - - - - - - - - - - - - - - - - - - ); -}; diff --git a/src/plugins/console/public/application/components/welcome_panel.tsx b/src/plugins/console/public/application/components/welcome_panel.tsx deleted file mode 100644 index 967d173821e65..0000000000000 --- a/src/plugins/console/public/application/components/welcome_panel.tsx +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import React from 'react'; -import { FormattedMessage } from '@kbn/i18n-react'; - -import { - EuiFlyout, - EuiFlyoutHeader, - EuiFlyoutBody, - EuiTitle, - EuiButton, - EuiText, - EuiFlyoutFooter, - EuiCode, -} from '@elastic/eui'; -import EditorExample from './editor_example'; -import * as examples from '../../../common/constants/welcome_panel'; - -interface Props { - onDismiss: () => void; -} - -export function WelcomePanel(props: Props) { - return ( - - - -

- -

- - - - -

- -

- -

- kbn:, - }} - /> -

- -

- -

-

- -

- - -

- -

-

- #, - doubleSlash: //, - slashAsterisk: /*, - asteriskSlash: */, - }} - /> -

- -

- -

-

- ${variableName}, - }} - /> -

- -
    -
  1. - Variables, - }} - /> -
    2. -
  2. - -
    3. -
- - - - - - - - - - ); -} diff --git a/src/plugins/console/public/application/containers/config/config.tsx b/src/plugins/console/public/application/containers/config/config.tsx new file mode 100644 index 0000000000000..503fdbd9c7354 --- /dev/null +++ b/src/plugins/console/public/application/containers/config/config.tsx @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import React from 'react'; +import { EuiFlexGroup, EuiFlexItem, EuiPanel, EuiSpacer } from '@elastic/eui'; + +import { Settings } from './settings'; +import { Variables } from './variables'; + +export interface Props { + isVerticalLayout: boolean; +} + +export function Config({ isVerticalLayout }: Props) { + return ( + + + + + + + + + + + + + ); +} diff --git a/src/plugins/console/public/application/containers/editor/utilities/index.ts b/src/plugins/console/public/application/containers/config/index.ts similarity index 93% rename from src/plugins/console/public/application/containers/editor/utilities/index.ts rename to src/plugins/console/public/application/containers/config/index.ts index 7561f02006235..b582701ab4481 100644 --- a/src/plugins/console/public/application/containers/editor/utilities/index.ts +++ b/src/plugins/console/public/application/containers/config/index.ts @@ -7,4 +7,4 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -export * from './output_data'; +export { Config } from './config'; diff --git a/src/plugins/console/public/application/containers/settings.tsx b/src/plugins/console/public/application/containers/config/settings.tsx similarity index 88% rename from src/plugins/console/public/application/containers/settings.tsx rename to src/plugins/console/public/application/containers/config/settings.tsx index 2c952f4c5d7f9..d5e10f4d2c337 100644 --- a/src/plugins/console/public/application/containers/settings.tsx +++ b/src/plugins/console/public/application/containers/config/settings.tsx @@ -9,11 +9,10 @@ import React from 'react'; -import { AutocompleteOptions, DevToolsSettingsModal } from '../components'; +import { AutocompleteOptions, SettingsEditor } from '../../components/settings'; -import { useServicesContext, useEditorActionContext } from '../contexts'; -import { DevToolsSettings, Settings as SettingsService } from '../../services'; -import type { SenseEditor } from '../models'; +import { useServicesContext, useEditorActionContext } from '../../contexts'; +import { DevToolsSettings, Settings as SettingsService } from '../../../services'; const getAutocompleteDiff = ( newSettings: DevToolsSettings, @@ -25,12 +24,7 @@ const getAutocompleteDiff = ( }) as AutocompleteOptions[]; }; -export interface Props { - onClose: () => void; - editorInstance: SenseEditor | null; -} - -export function Settings({ onClose, editorInstance }: Props) { +export function Settings() { const { services: { settings, autocompleteInfo }, } = useServicesContext(); @@ -92,18 +86,15 @@ export function Settings({ onClose, editorInstance }: Props) { type: 'updateSettings', payload: newSettings, }); - onClose(); }; return ( - refreshAutocompleteSettings(settings, selectedSettings) } settings={settings.toJSON()} - editorInstance={editorInstance} /> ); } diff --git a/src/plugins/console/public/application/containers/variables.tsx b/src/plugins/console/public/application/containers/config/variables.tsx similarity index 66% rename from src/plugins/console/public/application/containers/variables.tsx rename to src/plugins/console/public/application/containers/config/variables.tsx index 54e191d04a9de..32b9615f529aa 100644 --- a/src/plugins/console/public/application/containers/variables.tsx +++ b/src/plugins/console/public/application/containers/config/variables.tsx @@ -8,27 +8,22 @@ */ import React from 'react'; -import { DevToolsVariablesFlyout, DevToolsVariable } from '../components'; -import { useServicesContext } from '../contexts'; -import { StorageKeys } from '../../services'; -import { DEFAULT_VARIABLES } from '../../../common/constants'; +import { type DevToolsVariable, VariablesEditor } from '../../components/variables'; +import { useServicesContext } from '../../contexts'; +import { StorageKeys } from '../../../services'; +import { DEFAULT_VARIABLES } from '../../../../common/constants'; -interface VariablesProps { - onClose: () => void; -} - -export function Variables({ onClose }: VariablesProps) { +export function Variables() { const { services: { storage }, } = useServicesContext(); const onSaveVariables = (newVariables: DevToolsVariable[]) => { storage.set(StorageKeys.VARIABLES, newVariables); - onClose(); }; + return ( - diff --git a/src/plugins/console/public/application/containers/console_history/console_history.tsx b/src/plugins/console/public/application/containers/console_history/console_history.tsx deleted file mode 100644 index 220e0b6a998aa..0000000000000 --- a/src/plugins/console/public/application/containers/console_history/console_history.tsx +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import React, { useCallback, useEffect, useMemo, useState, useRef } from 'react'; -import { i18n } from '@kbn/i18n'; -import { memoize } from 'lodash'; -import moment from 'moment'; -import { - keys, - EuiSpacer, - EuiIcon, - EuiTitle, - EuiFlexItem, - EuiFlexGroup, - EuiButtonEmpty, - EuiButton, -} from '@elastic/eui'; - -import { useServicesContext } from '../../contexts'; -import { HistoryViewer } from './history_viewer'; -import { HistoryViewer as HistoryViewerMonaco } from './history_viewer_monaco'; -import { useEditorReadContext } from '../../contexts/editor_context'; -import { useRestoreRequestFromHistory } from '../../hooks'; - -interface Props { - close: () => void; -} - -const CHILD_ELEMENT_PREFIX = 'historyReq'; - -export function ConsoleHistory({ close }: Props) { - const { - services: { history }, - config: { isMonacoEnabled }, - } = useServicesContext(); - - const { settings: readOnlySettings } = useEditorReadContext(); - - const [requests, setPastRequests] = useState(history.getHistory()); - - const clearHistory = useCallback(() => { - history.clearHistory(); - setPastRequests(history.getHistory()); - }, [history]); - - const listRef = useRef(null); - - const [viewingReq, setViewingReq] = useState(null); - const [selectedIndex, setSelectedIndex] = useState(0); - const selectedReq = useRef(null); - - const describeReq = useMemo(() => { - const _describeReq = (req: { endpoint: string; time: string }) => { - const endpoint = req.endpoint; - const date = moment(req.time); - - let formattedDate = date.format('MMM D'); - if (date.diff(moment(), 'days') > -7) { - formattedDate = date.fromNow(); - } - - return `${endpoint} (${formattedDate})`; - }; - - (_describeReq as any).cache = new WeakMap(); - - return memoize(_describeReq); - }, []); - - const scrollIntoView = useCallback((idx: number) => { - const activeDescendant = listRef.current!.querySelector(`#${CHILD_ELEMENT_PREFIX}${idx}`); - if (activeDescendant) { - activeDescendant.scrollIntoView(); - } - }, []); - - const initialize = useCallback(() => { - const nextSelectedIndex = 0; - (describeReq as any).cache = new WeakMap(); - setViewingReq(requests[nextSelectedIndex]); - selectedReq.current = requests[nextSelectedIndex]; - setSelectedIndex(nextSelectedIndex); - scrollIntoView(nextSelectedIndex); - }, [describeReq, requests, scrollIntoView]); - - const clear = () => { - clearHistory(); - initialize(); - }; - - const restoreRequestFromHistory = useRestoreRequestFromHistory(isMonacoEnabled); - - useEffect(() => { - initialize(); - }, [initialize]); - - useEffect(() => { - const done = history.change(setPastRequests); - return () => done(); - }, [history]); - - /* eslint-disable jsx-a11y/no-noninteractive-element-to-interactive-role,jsx-a11y/click-events-have-key-events */ - return ( - <> -
- -

{i18n.translate('console.historyPage.pageTitle', { defaultMessage: 'History' })}

- - -
-
    { - if (ev.key === keys.ENTER) { - restoreRequestFromHistory(selectedReq.current); - return; - } - - let currentIdx = selectedIndex; - - if (ev.key === keys.ARROW_UP) { - ev.preventDefault(); - --currentIdx; - } else if (ev.key === keys.ARROW_DOWN) { - ev.preventDefault(); - ++currentIdx; - } - - const nextSelectedIndex = Math.min(Math.max(0, currentIdx), requests.length - 1); - - setViewingReq(requests[nextSelectedIndex]); - selectedReq.current = requests[nextSelectedIndex]; - setSelectedIndex(nextSelectedIndex); - scrollIntoView(nextSelectedIndex); - }} - role="listbox" - className="list-group conHistory__reqs" - tabIndex={0} - aria-activedescendant={`${CHILD_ELEMENT_PREFIX}${selectedIndex}`} - aria-label={i18n.translate('console.historyPage.requestListAriaLabel', { - defaultMessage: 'History of sent requests', - })} - > - {requests.map((req, idx) => { - const reqDescription = describeReq(req); - const isSelected = viewingReq === req; - return ( - // Ignore a11y issues on li's -
  • { - setViewingReq(req); - selectedReq.current = req; - setSelectedIndex(idx); - }} - role="option" - onMouseEnter={() => setViewingReq(req)} - onMouseLeave={() => setViewingReq(selectedReq.current)} - onDoubleClick={() => restoreRequestFromHistory(selectedReq.current)} - aria-label={i18n.translate('console.historyPage.itemOfRequestListAriaLabel', { - defaultMessage: 'Request: {historyItem}', - values: { historyItem: reqDescription }, - })} - aria-selected={isSelected} - > - {reqDescription} - - - -
    • - ); - })} -
- -
- - {isMonacoEnabled ? ( - - ) : ( - - )} -
- - - - - - clear()} - > - {i18n.translate('console.historyPage.clearHistoryButtonLabel', { - defaultMessage: 'Clear', - })} - - - - - - - close()} - > - {i18n.translate('console.historyPage.closehistoryButtonLabel', { - defaultMessage: 'Close', - })} - - - - - restoreRequestFromHistory(selectedReq.current)} - > - {i18n.translate('console.historyPage.applyHistoryButtonLabel', { - defaultMessage: 'Apply', - })} - - - - - -
- - - ); -} diff --git a/src/plugins/console/public/application/containers/console_history/history_viewer.tsx b/src/plugins/console/public/application/containers/console_history/history_viewer.tsx deleted file mode 100644 index 92d58e557cd89..0000000000000 --- a/src/plugins/console/public/application/containers/console_history/history_viewer.tsx +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import React, { useEffect, useRef } from 'react'; -import { i18n } from '@kbn/i18n'; - -import { DevToolsSettings } from '../../../services'; -import { subscribeResizeChecker } from '../editor/legacy/subscribe_console_resize_checker'; - -import * as InputMode from '../../models/legacy_core_editor/mode/input'; -const inputMode = new InputMode.Mode(); -import * as editor from '../../models/legacy_core_editor'; -import { applyCurrentSettings } from '../editor/legacy/console_editor/apply_editor_settings'; -import { formatRequestBodyDoc } from '../../../lib/utils'; - -interface Props { - settings: DevToolsSettings; - req: { method: string; endpoint: string; data: string; time: string } | null; -} - -export function HistoryViewer({ settings, req }: Props) { - const divRef = useRef(null); - const viewerRef = useRef(null); - - useEffect(() => { - const viewer = editor.createReadOnlyAceEditor(divRef.current!); - viewerRef.current = viewer; - const unsubscribe = subscribeResizeChecker(divRef.current!, viewer); - return () => unsubscribe(); - }, []); - - useEffect(() => { - applyCurrentSettings(viewerRef.current!, settings); - }, [settings]); - - if (viewerRef.current) { - const { current: viewer } = viewerRef; - if (req) { - const indent = true; - const formattedData = req.data ? formatRequestBodyDoc([req.data], indent).data : ''; - const s = req.method + ' ' + req.endpoint + '\n' + formattedData; - viewer.update(s, inputMode); - viewer.clearSelection(); - } else { - viewer.update( - i18n.translate('console.historyPage.noHistoryTextMessage', { - defaultMessage: 'No history available', - }), - inputMode - ); - } - } - - return
; -} diff --git a/src/plugins/console/public/application/containers/editor/monaco/components/context_menu/context_menu.tsx b/src/plugins/console/public/application/containers/editor/components/context_menu/context_menu.tsx similarity index 90% rename from src/plugins/console/public/application/containers/editor/monaco/components/context_menu/context_menu.tsx rename to src/plugins/console/public/application/containers/editor/components/context_menu/context_menu.tsx index 34001018900b5..3860f0b7bc704 100644 --- a/src/plugins/console/public/application/containers/editor/monaco/components/context_menu/context_menu.tsx +++ b/src/plugins/console/public/application/containers/editor/components/context_menu/context_menu.tsx @@ -9,7 +9,7 @@ import React, { useState } from 'react'; import { - EuiIcon, + EuiButtonIcon, EuiContextMenuPanel, EuiContextMenuItem, EuiPopover, @@ -18,16 +18,17 @@ import { EuiLink, EuiLoadingSpinner, } from '@elastic/eui'; +import { css } from '@emotion/react'; import { NotificationsSetup } from '@kbn/core/public'; import { FormattedMessage } from '@kbn/i18n-react'; import { i18n } from '@kbn/i18n'; import { LanguageSelectorModal } from './language_selector_modal'; -import { convertRequestToLanguage } from '../../../../../../services'; +import { convertRequestToLanguage } from '../../../../../services'; import type { EditorRequest } from '../../types'; -import { useServicesContext } from '../../../../../contexts'; -import { StorageKeys } from '../../../../../../services'; -import { DEFAULT_LANGUAGE, AVAILABLE_LANGUAGES } from '../../../../../../../common/constants'; +import { useServicesContext } from '../../../../contexts'; +import { StorageKeys } from '../../../../../services'; +import { DEFAULT_LANGUAGE, AVAILABLE_LANGUAGES } from '../../../../../../common/constants'; interface Props { getRequests: () => Promise; @@ -36,6 +37,20 @@ interface Props { notifications: NotificationsSetup; } +const styles = { + // Remove the default underline on hover for the context menu items since it + // will also be applied to the language selector button, and apply it only to + // the text in the context menu item. + button: css` + &:hover { + text-decoration: none !important; + .languageSelector { + text-decoration: underline; + } + } + `, +}; + const DELAY_FOR_HIDING_SPINNER = 500; const getLanguageLabelByValue = (value: string) => { @@ -158,15 +173,15 @@ export const ContextMenu = ({ }; const button = ( - setIsPopoverOpen((prev) => !prev)} data-test-subj="toggleConsoleMenu" aria-label={i18n.translate('console.requestOptionsButtonAriaLabel', { defaultMessage: 'Request options', })} - > - - + iconType="boxesVertical" + iconSize="s" + /> ); const items = [ @@ -187,10 +202,11 @@ export const ContextMenu = ({ onCopyAsSubmit(); }} icon="copyClipboard" + css={styles.button} > - + void; diff --git a/src/plugins/console/public/application/containers/editor/monaco/components/index.ts b/src/plugins/console/public/application/containers/editor/components/index.ts similarity index 100% rename from src/plugins/console/public/application/containers/editor/monaco/components/index.ts rename to src/plugins/console/public/application/containers/editor/components/index.ts diff --git a/src/plugins/console/public/application/containers/editor/editor.tsx b/src/plugins/console/public/application/containers/editor/editor.tsx index 3eff2d97b3499..c999deee78637 100644 --- a/src/plugins/console/public/application/containers/editor/editor.tsx +++ b/src/plugins/console/public/application/containers/editor/editor.tsx @@ -7,95 +7,283 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import React, { useCallback, memo, useEffect, useState } from 'react'; +import React, { useRef, useCallback, memo, useEffect, useState } from 'react'; import { debounce } from 'lodash'; -import { EuiProgress } from '@elastic/eui'; +import { + EuiProgress, + EuiSplitPanel, + EuiFlexGroup, + EuiFlexItem, + EuiButtonEmpty, + EuiResizableContainer, +} from '@elastic/eui'; +import { euiThemeVars } from '@kbn/ui-theme'; -import { EditorContentSpinner } from '../../components'; -import { Panel, PanelsContainer } from '..'; -import { Editor as EditorUI, EditorOutput } from './legacy/console_editor'; +import { i18n } from '@kbn/i18n'; +import { TextObject } from '../../../../common/text_object'; + +import { + EditorContentSpinner, + OutputPanelEmptyState, + NetworkRequestStatusBar, +} from '../../components'; import { getAutocompleteInfo, StorageKeys } from '../../../services'; -import { useEditorReadContext, useServicesContext, useRequestReadContext } from '../../contexts'; -import type { SenseEditor } from '../../models'; -import { MonacoEditor, MonacoEditorOutput } from './monaco'; +import { + useEditorReadContext, + useServicesContext, + useRequestReadContext, + useRequestActionContext, + useEditorActionContext, +} from '../../contexts'; +import { MonacoEditor } from './monaco_editor'; +import { MonacoEditorOutput } from './monaco_editor_output'; +import { getResponseWithMostSevereStatusCode } from '../../../lib/utils'; -const INITIAL_PANEL_WIDTH = 50; -const PANEL_MIN_WIDTH = '100px'; +const INITIAL_PANEL_SIZE = 50; +const PANEL_MIN_SIZE = '20%'; +const DEBOUNCE_DELAY = 500; interface Props { loading: boolean; - setEditorInstance: (instance: SenseEditor) => void; + isVerticalLayout: boolean; + inputEditorValue: string; + setInputEditorValue: (value: string) => void; } -export const Editor = memo(({ loading, setEditorInstance }: Props) => { - const { - services: { storage }, - config: { isMonacoEnabled } = {}, - } = useServicesContext(); - - const { currentTextObject } = useEditorReadContext(); - const { requestInFlight } = useRequestReadContext(); - - const [fetchingMappings, setFetchingMappings] = useState(false); - - useEffect(() => { - const subscription = getAutocompleteInfo().mapping.isLoading$.subscribe(setFetchingMappings); - return () => { - subscription.unsubscribe(); - }; - }, []); - - const [firstPanelWidth, secondPanelWidth] = storage.get(StorageKeys.WIDTH, [ - INITIAL_PANEL_WIDTH, - INITIAL_PANEL_WIDTH, - ]); - - /* eslint-disable-next-line react-hooks/exhaustive-deps */ - const onPanelWidthChange = useCallback( - debounce((widths: number[]) => { - storage.set(StorageKeys.WIDTH, widths); - }, 300), - [] - ); - - if (!currentTextObject) return null; - - return ( - <> - {requestInFlight || fetchingMappings ? ( -
- -
- ) : null} - - - {loading ? ( - - ) : isMonacoEnabled ? ( - - ) : ( - - )} - - { + const { + services: { storage, objectStorageClient }, + } = useServicesContext(); + + const editorValueRef = useRef(null); + const { currentTextObject } = useEditorReadContext(); + const { + requestInFlight, + lastResult: { data: requestData, error: requestError }, + } = useRequestReadContext(); + + const dispatch = useRequestActionContext(); + const editorDispatch = useEditorActionContext(); + + const [fetchingAutocompleteEntities, setFetchingAutocompleteEntities] = useState(false); + + useEffect(() => { + const debouncedSetFechingAutocompleteEntities = debounce( + setFetchingAutocompleteEntities, + DEBOUNCE_DELAY + ); + const subscription = getAutocompleteInfo().isLoading$.subscribe( + debouncedSetFechingAutocompleteEntities + ); + + return () => { + subscription.unsubscribe(); + debouncedSetFechingAutocompleteEntities.cancel(); + }; + }, []); + + const [firstPanelSize, secondPanelSize] = storage.get(StorageKeys.SIZE, [ + INITIAL_PANEL_SIZE, + INITIAL_PANEL_SIZE, + ]); + + /* eslint-disable-next-line react-hooks/exhaustive-deps */ + const onPanelSizeChange = useCallback( + debounce((sizes) => { + storage.set(StorageKeys.SIZE, Object.values(sizes)); + }, 300), + [] + ); + + /* eslint-disable-next-line react-hooks/exhaustive-deps */ + const debouncedUpdateLocalStorageValue = useCallback( + debounce((textObject: TextObject) => { + editorValueRef.current = textObject; + objectStorageClient.text.update(textObject); + }, DEBOUNCE_DELAY), + [] + ); + + useEffect(() => { + return () => { + editorDispatch({ + type: 'setCurrentTextObject', + payload: editorValueRef.current!, + }); + }; + }, [editorDispatch]); + + // Always keep the localstorage in sync with the value in the editor + // to avoid losing the text object when the user navigates away from the shell + useEffect(() => { + // Only update when its not empty, this is to avoid setting the localstorage value + // to an empty string that will then be replaced by the example request. + if (inputEditorValue !== '') { + const textObject = { + ...currentTextObject, + text: inputEditorValue, + updatedAt: Date.now(), + } as TextObject; + + debouncedUpdateLocalStorageValue(textObject); + } + /* eslint-disable-next-line react-hooks/exhaustive-deps */ + }, [inputEditorValue, debouncedUpdateLocalStorageValue]); + + const data = getResponseWithMostSevereStatusCode(requestData) ?? requestError; + const isLoading = loading || requestInFlight; + + if (!currentTextObject) return null; + + return ( + <> + {fetchingAutocompleteEntities ? ( +
+ +
+ ) : null} + onPanelSizeChange(sizes)} + data-test-subj="consoleEditorContainer" > - {loading ? ( - - ) : isMonacoEnabled ? ( - - ) : ( - + {(EuiResizablePanel, EuiResizableButton) => ( + <> + + + + {loading ? ( + + ) : ( + + )} + + + {!loading && ( + + setInputEditorValue('')} + > + {i18n.translate('console.editor.clearConsoleInputButton', { + defaultMessage: 'Clear this input', + })} + + + )} + + + + + + + + + {data ? ( + + ) : isLoading ? ( + + ) : ( + + )} + + + {(data || isLoading) && ( + + + + dispatch({ type: 'cleanRequest', payload: undefined })} + > + {i18n.translate('console.editor.clearConsoleOutputButton', { + defaultMessage: 'Clear this output', + })} + + + + + + + + + )} + + + )} - - - - ); -}); + + + ); + } +); diff --git a/src/plugins/console/public/application/containers/editor/monaco/hooks/index.ts b/src/plugins/console/public/application/containers/editor/hooks/index.ts similarity index 100% rename from src/plugins/console/public/application/containers/editor/monaco/hooks/index.ts rename to src/plugins/console/public/application/containers/editor/hooks/index.ts diff --git a/src/plugins/console/public/application/containers/editor/monaco/hooks/use_register_keyboard_commands.ts b/src/plugins/console/public/application/containers/editor/hooks/use_register_keyboard_commands.ts similarity index 100% rename from src/plugins/console/public/application/containers/editor/monaco/hooks/use_register_keyboard_commands.ts rename to src/plugins/console/public/application/containers/editor/hooks/use_register_keyboard_commands.ts diff --git a/src/plugins/console/public/application/containers/editor/monaco/hooks/use_resize_checker_utils.ts b/src/plugins/console/public/application/containers/editor/hooks/use_resize_checker_utils.ts similarity index 100% rename from src/plugins/console/public/application/containers/editor/monaco/hooks/use_resize_checker_utils.ts rename to src/plugins/console/public/application/containers/editor/hooks/use_resize_checker_utils.ts diff --git a/src/plugins/console/public/application/containers/editor/monaco/hooks/use_set_initial_value.ts b/src/plugins/console/public/application/containers/editor/hooks/use_set_initial_value.ts similarity index 91% rename from src/plugins/console/public/application/containers/editor/monaco/hooks/use_set_initial_value.ts rename to src/plugins/console/public/application/containers/editor/hooks/use_set_initial_value.ts index 41a3b77a105cd..961ea586bc291 100644 --- a/src/plugins/console/public/application/containers/editor/monaco/hooks/use_set_initial_value.ts +++ b/src/plugins/console/public/application/containers/editor/hooks/use_set_initial_value.ts @@ -13,7 +13,7 @@ import { IToasts } from '@kbn/core-notifications-browser'; import { decompressFromEncodedURIComponent } from 'lz-string'; import { i18n } from '@kbn/i18n'; import { useEffect } from 'react'; -import { DEFAULT_INPUT_VALUE } from '../../../../../../common/constants'; +import { DEFAULT_INPUT_VALUE } from '../../../../../common/constants'; interface QueryParams { load_from: string; @@ -21,7 +21,7 @@ interface QueryParams { interface SetInitialValueParams { /** The text value that is initially in the console editor. */ - initialTextValue?: string; + localStorageValue?: string; /** The function that sets the state of the value in the console editor. */ setValue: (value: string) => void; /** The toasts service. */ @@ -45,7 +45,7 @@ export const readLoadFromParam = () => { * @param params The {@link SetInitialValueParams} to use. */ export const useSetInitialValue = (params: SetInitialValueParams) => { - const { initialTextValue, setValue, toasts } = params; + const { localStorageValue, setValue, toasts } = params; useEffect(() => { const loadBufferFromRemote = async (url: string) => { @@ -61,7 +61,7 @@ export const useSetInitialValue = (params: SetInitialValueParams) => { if (parsedURL.origin === 'https://www.elastic.co') { const resp = await fetch(parsedURL); const data = await resp.text(); - setValue(`${initialTextValue}\n\n${data}`); + setValue(`${localStorageValue}\n\n${data}`); } else { toasts.addWarning( i18n.translate('console.monaco.loadFromDataUnrecognizedUrlErrorMessage', { @@ -107,11 +107,11 @@ export const useSetInitialValue = (params: SetInitialValueParams) => { if (loadFromParam) { loadBufferFromRemote(loadFromParam); } else { - setValue(initialTextValue || DEFAULT_INPUT_VALUE); + setValue(localStorageValue || DEFAULT_INPUT_VALUE); } return () => { window.removeEventListener('hashchange', onHashChange); }; - }, [initialTextValue, setValue, toasts]); + }, [localStorageValue, setValue, toasts]); }; diff --git a/src/plugins/console/public/application/containers/editor/monaco/hooks/use_setup_autocomplete_polling.ts b/src/plugins/console/public/application/containers/editor/hooks/use_setup_autocomplete_polling.ts similarity index 94% rename from src/plugins/console/public/application/containers/editor/monaco/hooks/use_setup_autocomplete_polling.ts rename to src/plugins/console/public/application/containers/editor/hooks/use_setup_autocomplete_polling.ts index 1a1a5bb77dd1e..4785be4054ee0 100644 --- a/src/plugins/console/public/application/containers/editor/monaco/hooks/use_setup_autocomplete_polling.ts +++ b/src/plugins/console/public/application/containers/editor/hooks/use_setup_autocomplete_polling.ts @@ -8,7 +8,7 @@ */ import { useEffect } from 'react'; -import { AutocompleteInfo, Settings } from '../../../../../services'; +import { AutocompleteInfo, Settings } from '../../../../services'; interface SetupAutocompletePollingParams { /** The Console autocomplete service. */ diff --git a/src/plugins/console/public/application/containers/editor/monaco/hooks/use_setup_autosave.ts b/src/plugins/console/public/application/containers/editor/hooks/use_setup_autosave.ts similarity index 96% rename from src/plugins/console/public/application/containers/editor/monaco/hooks/use_setup_autosave.ts rename to src/plugins/console/public/application/containers/editor/hooks/use_setup_autosave.ts index 6f46b8ce6589d..8b4bfaa888649 100644 --- a/src/plugins/console/public/application/containers/editor/monaco/hooks/use_setup_autosave.ts +++ b/src/plugins/console/public/application/containers/editor/hooks/use_setup_autosave.ts @@ -8,7 +8,7 @@ */ import { useEffect, useRef } from 'react'; -import { useSaveCurrentTextObject } from '../../../../hooks'; +import { useSaveCurrentTextObject } from '../../../hooks'; import { readLoadFromParam } from './use_set_initial_value'; interface SetupAutosaveParams { diff --git a/src/plugins/console/public/application/containers/editor/index.ts b/src/plugins/console/public/application/containers/editor/index.ts index c9fbe97f01d8d..696806097badd 100644 --- a/src/plugins/console/public/application/containers/editor/index.ts +++ b/src/plugins/console/public/application/containers/editor/index.ts @@ -7,5 +7,4 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -export { autoIndent, getDocumentation } from './legacy'; export { Editor } from './editor'; diff --git a/src/plugins/console/public/application/containers/editor/legacy/console_editor/apply_editor_settings.ts b/src/plugins/console/public/application/containers/editor/legacy/console_editor/apply_editor_settings.ts deleted file mode 100644 index 75e2516a52a7a..0000000000000 --- a/src/plugins/console/public/application/containers/editor/legacy/console_editor/apply_editor_settings.ts +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import { DevToolsSettings } from '../../../../../services'; -import { CoreEditor } from '../../../../../types'; -import { CustomAceEditor } from '../../../../models/legacy_core_editor'; - -export function applyCurrentSettings( - editor: CoreEditor | CustomAceEditor, - settings: DevToolsSettings -) { - if ((editor as { setStyles?: Function }).setStyles) { - (editor as CoreEditor).setStyles({ - wrapLines: settings.wrapMode, - fontSize: settings.fontSize + 'px', - }); - } else { - (editor as CustomAceEditor).getSession().setUseWrapMode(settings.wrapMode); - (editor as CustomAceEditor).container.style.fontSize = settings.fontSize + 'px'; - } -} diff --git a/src/plugins/console/public/application/containers/editor/legacy/console_editor/editor.test.mock.tsx b/src/plugins/console/public/application/containers/editor/legacy/console_editor/editor.test.mock.tsx deleted file mode 100644 index f0371562a77bb..0000000000000 --- a/src/plugins/console/public/application/containers/editor/legacy/console_editor/editor.test.mock.tsx +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -// TODO(jbudz): should be removed when upgrading to TS@4.8 -// this is a skip for the errors created when typechecking with isolatedModules -export {}; - -jest.mock('../../../../contexts/editor_context/editor_registry', () => ({ - instance: { - setInputEditor: () => {}, - getInputEditor: () => ({ - getRequestsInRange: async () => [{ test: 'test' }], - getCoreEditor: () => ({ getCurrentPosition: jest.fn() }), - }), - }, -})); -jest.mock('../../../../components/editor_example', () => {}); -jest.mock('../../../../models/sense_editor', () => { - return { - create: () => ({ - getCoreEditor: () => ({ - registerKeyboardShortcut: jest.fn(), - setStyles: jest.fn(), - getContainer: () => ({ - focus: () => {}, - }), - on: jest.fn(), - addFoldsAtRanges: jest.fn(), - getAllFoldRanges: jest.fn(), - }), - update: jest.fn(), - commands: { - addCommand: () => {}, - }, - }), - }; -}); - -jest.mock('../../../../hooks/use_send_current_request/send_request', () => ({ - sendRequest: jest.fn(), -})); -jest.mock('../../../../../lib/autocomplete/get_endpoint_from_position', () => ({ - getEndpointFromPosition: jest.fn(), -})); diff --git a/src/plugins/console/public/application/containers/editor/legacy/console_editor/editor.test.tsx b/src/plugins/console/public/application/containers/editor/legacy/console_editor/editor.test.tsx deleted file mode 100644 index 589be10596b9b..0000000000000 --- a/src/plugins/console/public/application/containers/editor/legacy/console_editor/editor.test.tsx +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -jest.mock('../../../../../lib/utils', () => ({ replaceVariables: jest.fn() })); - -import './editor.test.mock'; - -import React from 'react'; -import { mount } from 'enzyme'; -import { I18nProvider } from '@kbn/i18n-react'; -import { act } from 'react-dom/test-utils'; -import * as sinon from 'sinon'; - -import { serviceContextMock } from '../../../../contexts/services_context.mock'; - -import { nextTick } from '@kbn/test-jest-helpers'; -import { - ServicesContextProvider, - EditorContextProvider, - RequestContextProvider, - ContextValue, -} from '../../../../contexts'; - -// Mocked functions -import { sendRequest } from '../../../../hooks/use_send_current_request/send_request'; -import { getEndpointFromPosition } from '../../../../../lib/autocomplete/get_endpoint_from_position'; -import type { DevToolsSettings } from '../../../../../services'; -import * as consoleMenuActions from '../console_menu_actions'; -import { Editor } from './editor'; -import * as utils from '../../../../../lib/utils'; - -describe('Legacy (Ace) Console Editor Component Smoke Test', () => { - let mockedAppContextValue: ContextValue; - const sandbox = sinon.createSandbox(); - - const doMount = () => - mount( - - - - - {}} /> - - - - - ); - - beforeEach(() => { - document.queryCommandSupported = sinon.fake(() => true); - mockedAppContextValue = serviceContextMock.create(); - (utils.replaceVariables as jest.Mock).mockReturnValue(['test']); - }); - - afterEach(() => { - jest.clearAllMocks(); - sandbox.restore(); - }); - - it('calls send current request', async () => { - (getEndpointFromPosition as jest.Mock).mockReturnValue({ patterns: [] }); - (sendRequest as jest.Mock).mockRejectedValue({}); - const editor = doMount(); - act(() => { - editor.find('button[data-test-subj~="sendRequestButton"]').simulate('click'); - }); - await nextTick(); - expect(sendRequest).toBeCalledTimes(1); - }); - - it('opens docs', () => { - const stub = sandbox.stub(consoleMenuActions, 'getDocumentation'); - const editor = doMount(); - const consoleMenuToggle = editor.find('[data-test-subj~="toggleConsoleMenu"]').last(); - consoleMenuToggle.simulate('click'); - - const docsButton = editor.find('[data-test-subj~="consoleMenuOpenDocs"]').last(); - docsButton.simulate('click'); - - expect(stub.callCount).toBe(1); - }); - - it('prompts auto-indent', () => { - const stub = sandbox.stub(consoleMenuActions, 'autoIndent'); - const editor = doMount(); - const consoleMenuToggle = editor.find('[data-test-subj~="toggleConsoleMenu"]').last(); - consoleMenuToggle.simulate('click'); - - const autoIndentButton = editor.find('[data-test-subj~="consoleMenuAutoIndent"]').last(); - autoIndentButton.simulate('click'); - - expect(stub.callCount).toBe(1); - }); -}); diff --git a/src/plugins/console/public/application/containers/editor/legacy/console_editor/editor.tsx b/src/plugins/console/public/application/containers/editor/legacy/console_editor/editor.tsx deleted file mode 100644 index a0119ac2ec8fa..0000000000000 --- a/src/plugins/console/public/application/containers/editor/legacy/console_editor/editor.tsx +++ /dev/null @@ -1,343 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import { - EuiFlexGroup, - EuiFlexItem, - EuiIcon, - EuiLink, - EuiScreenReaderOnly, - EuiToolTip, -} from '@elastic/eui'; -import { i18n } from '@kbn/i18n'; -import { debounce } from 'lodash'; -import { decompressFromEncodedURIComponent } from 'lz-string'; -import { parse } from 'query-string'; -import React, { CSSProperties, useCallback, useEffect, useRef, useState } from 'react'; -import { ace } from '@kbn/es-ui-shared-plugin/public'; -import { ConsoleMenu } from '../../../../components'; -import { useEditorReadContext, useServicesContext } from '../../../../contexts'; -import { - useSaveCurrentTextObject, - useSendCurrentRequest, - useSetInputEditor, -} from '../../../../hooks'; -import * as senseEditor from '../../../../models/sense_editor'; -import { autoIndent, getDocumentation } from '../console_menu_actions'; -import { subscribeResizeChecker } from '../subscribe_console_resize_checker'; -import { applyCurrentSettings } from './apply_editor_settings'; -import { registerCommands } from './keyboard_shortcuts'; -import type { SenseEditor } from '../../../../models/sense_editor'; -import { StorageKeys } from '../../../../../services'; -import { DEFAULT_INPUT_VALUE } from '../../../../../../common/constants'; - -const { useUIAceKeyboardMode } = ace; - -export interface EditorProps { - initialTextValue: string; - setEditorInstance: (instance: SenseEditor) => void; -} - -interface QueryParams { - load_from: string; -} - -const abs: CSSProperties = { - position: 'absolute', - top: '0', - left: '0', - bottom: '0', - right: '0', -}; - -const inputId = 'ConAppInputTextarea'; - -function EditorUI({ initialTextValue, setEditorInstance }: EditorProps) { - const { - services: { - history, - notifications, - settings: settingsService, - esHostService, - http, - autocompleteInfo, - storage, - }, - docLinkVersion, - ...startServices - } = useServicesContext(); - - const { settings } = useEditorReadContext(); - const setInputEditor = useSetInputEditor(); - const sendCurrentRequest = useSendCurrentRequest(); - const saveCurrentTextObject = useSaveCurrentTextObject(); - - const editorRef = useRef(null); - const editorInstanceRef = useRef(null); - - const [textArea, setTextArea] = useState(null); - useUIAceKeyboardMode(textArea, startServices, settings.isAccessibilityOverlayEnabled); - - const openDocumentation = useCallback(async () => { - const documentation = await getDocumentation(editorInstanceRef.current!, docLinkVersion); - if (!documentation) { - return; - } - window.open(documentation, '_blank'); - }, [docLinkVersion]); - - useEffect(() => { - editorInstanceRef.current = senseEditor.create(editorRef.current!); - const editor = editorInstanceRef.current; - const textareaElement = editorRef.current!.querySelector('textarea'); - - if (textareaElement) { - textareaElement.setAttribute('id', inputId); - textareaElement.setAttribute('data-test-subj', 'console-textarea'); - } - - const readQueryParams = () => { - const [, queryString] = (window.location.hash || window.location.search || '').split('?'); - - return parse(queryString || '', { sort: false }) as Required; - }; - - const loadBufferFromRemote = (url: string) => { - const coreEditor = editor.getCoreEditor(); - // Normalize and encode the URL to avoid issues with spaces and other special characters. - const encodedUrl = new URL(url).toString(); - if (/^https?:\/\//.test(encodedUrl)) { - const loadFrom: Record = { - url, - // Having dataType here is required as it doesn't allow jQuery to `eval` content - // coming from the external source thereby preventing XSS attack. - dataType: 'text', - kbnXsrfToken: false, - }; - - if (/https?:\/\/api\.github\.com/.test(url)) { - loadFrom.headers = { Accept: 'application/vnd.github.v3.raw' }; - } - - // Fire and forget. - $.ajax(loadFrom).done(async (data) => { - // when we load data from another Api we also must pass history - await editor.update(`${initialTextValue}\n ${data}`, true); - editor.moveToNextRequestEdge(false); - coreEditor.clearSelection(); - editor.highlightCurrentRequestsAndUpdateActionBar(); - coreEditor.getContainer().focus(); - }); - } - - // If we have a data URI instead of HTTP, LZ-decode it. This enables - // opening requests in Console from anywhere in Kibana. - if (/^data:/.test(url)) { - const data = decompressFromEncodedURIComponent(url.replace(/^data:text\/plain,/, '')); - - // Show a toast if we have a failure - if (data === null || data === '') { - notifications.toasts.addWarning( - i18n.translate('console.loadFromDataUriErrorMessage', { - defaultMessage: 'Unable to load data from the load_from query parameter in the URL', - }) - ); - return; - } - - editor.update(data, true); - editor.moveToNextRequestEdge(false); - coreEditor.clearSelection(); - editor.highlightCurrentRequestsAndUpdateActionBar(); - coreEditor.getContainer().focus(); - } - }; - - // Support for loading a console snippet from a remote source, like support docs. - const onHashChange = debounce(() => { - const { load_from: url } = readQueryParams(); - if (!url) { - return; - } - loadBufferFromRemote(url); - }, 200); - window.addEventListener('hashchange', onHashChange); - - const initialQueryParams = readQueryParams(); - - if (initialQueryParams.load_from) { - loadBufferFromRemote(initialQueryParams.load_from); - } else { - editor.update(initialTextValue || DEFAULT_INPUT_VALUE); - } - - function setupAutosave() { - let timer: number; - const saveDelay = 500; - - editor.getCoreEditor().on('change', () => { - if (timer) { - clearTimeout(timer); - } - timer = window.setTimeout(saveCurrentState, saveDelay); - }); - } - - function saveCurrentState() { - try { - const content = editor.getCoreEditor().getValue(); - saveCurrentTextObject(content); - } catch (e) { - // Ignoring saving error - } - } - - function restoreFolds() { - if (editor) { - const foldRanges = storage.get(StorageKeys.FOLDS, []); - editor.getCoreEditor().addFoldsAtRanges(foldRanges); - } - } - - restoreFolds(); - - function saveFoldsOnChange() { - if (editor) { - editor.getCoreEditor().on('changeFold', () => { - const foldRanges = editor.getCoreEditor().getAllFoldRanges(); - storage.set(StorageKeys.FOLDS, foldRanges); - }); - } - } - - saveFoldsOnChange(); - - setInputEditor(editor); - setTextArea(editorRef.current!.querySelector('textarea')); - - autocompleteInfo.retrieve(settingsService, settingsService.getAutocomplete()); - - const unsubscribeResizer = subscribeResizeChecker(editorRef.current!, editor); - if (!initialQueryParams.load_from) { - // Don't setup autosaving editor content when we pre-load content - // This prevents losing the user's current console content when - // `loadFrom` query param is used for a console session - setupAutosave(); - } - - return () => { - unsubscribeResizer(); - autocompleteInfo.clearSubscriptions(); - window.removeEventListener('hashchange', onHashChange); - if (editorInstanceRef.current) { - // Close autocomplete popup on unmount - editorInstanceRef.current?.getCoreEditor().detachCompleter(); - editorInstanceRef.current.getCoreEditor().destroy(); - } - }; - }, [ - notifications.toasts, - saveCurrentTextObject, - initialTextValue, - history, - setInputEditor, - settingsService, - http, - autocompleteInfo, - storage, - ]); - - useEffect(() => { - const { current: editor } = editorInstanceRef; - applyCurrentSettings(editor!.getCoreEditor(), settings); - // Preserve legacy focus behavior after settings have updated. - editor!.getCoreEditor().getContainer().focus(); - }, [settings]); - - useEffect(() => { - const { isKeyboardShortcutsEnabled } = settings; - if (isKeyboardShortcutsEnabled) { - registerCommands({ - senseEditor: editorInstanceRef.current!, - sendCurrentRequest, - openDocumentation, - }); - } - }, [openDocumentation, settings, sendCurrentRequest]); - - useEffect(() => { - const { current: editor } = editorInstanceRef; - if (editor) { - setEditorInstance(editor); - } - }, [setEditorInstance]); - - return ( -
-
-
    - - - - - - - - - - { - return editorInstanceRef.current!.getRequestsAsCURL(esHostService.getHost()); - }} - getDocumentation={() => { - return getDocumentation(editorInstanceRef.current!, docLinkVersion); - }} - autoIndent={(event) => { - autoIndent(editorInstanceRef.current!, event); - }} - notifications={notifications} - /> - - - - - - -
    -
    -
- ); -} - -export const Editor = React.memo(EditorUI); diff --git a/src/plugins/console/public/application/containers/editor/legacy/console_editor/editor_output.tsx b/src/plugins/console/public/application/containers/editor/legacy/console_editor/editor_output.tsx deleted file mode 100644 index 09cdf02cbab98..0000000000000 --- a/src/plugins/console/public/application/containers/editor/legacy/console_editor/editor_output.tsx +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import { VectorTile } from '@mapbox/vector-tile'; -import Protobuf from 'pbf'; -import { EuiScreenReaderOnly } from '@elastic/eui'; -import { i18n } from '@kbn/i18n'; -import React, { useEffect, useRef } from 'react'; -import { convertMapboxVectorTileToJson } from './mapbox_vector_tile'; -import { Mode } from '../../../../models/legacy_core_editor/mode/output'; - -// Ensure the modes we might switch to dynamically are available -import 'brace/mode/text'; -import 'brace/mode/hjson'; -import 'brace/mode/yaml'; - -import { - useEditorReadContext, - useRequestReadContext, - useServicesContext, -} from '../../../../contexts'; -import { createReadOnlyAceEditor, CustomAceEditor } from '../../../../models/legacy_core_editor'; -import { subscribeResizeChecker } from '../subscribe_console_resize_checker'; -import { applyCurrentSettings } from './apply_editor_settings'; -import { isJSONContentType, isMapboxVectorTile, safeExpandLiteralStrings } from '../../utilities'; - -function modeForContentType(contentType?: string) { - if (!contentType) { - return 'ace/mode/text'; - } - if (isJSONContentType(contentType) || isMapboxVectorTile(contentType)) { - // Using hjson will allow us to use comments in editor output and solves the problem with error markers - return 'ace/mode/hjson'; - } else if (contentType.indexOf('application/yaml') >= 0) { - return 'ace/mode/yaml'; - } - return 'ace/mode/text'; -} - -function EditorOutputUI() { - const editorRef = useRef(null); - const editorInstanceRef = useRef(null); - const { services } = useServicesContext(); - const { settings: readOnlySettings } = useEditorReadContext(); - const { - lastResult: { data, error }, - } = useRequestReadContext(); - const inputId = 'ConAppOutputTextarea'; - - useEffect(() => { - editorInstanceRef.current = createReadOnlyAceEditor(editorRef.current!); - const unsubscribe = subscribeResizeChecker(editorRef.current!, editorInstanceRef.current); - const textarea = editorRef.current!.querySelector('textarea')!; - textarea.setAttribute('id', inputId); - textarea.setAttribute('readonly', 'true'); - - return () => { - unsubscribe(); - editorInstanceRef.current!.destroy(); - }; - }, [services.settings]); - - useEffect(() => { - const editor = editorInstanceRef.current!; - if (data) { - const isMultipleRequest = data.length > 1; - const mode = isMultipleRequest - ? new Mode() - : modeForContentType(data[0].response.contentType); - editor.update( - data - .map((result) => { - const { value, contentType } = result.response; - - let editorOutput; - if (readOnlySettings.tripleQuotes && isJSONContentType(contentType)) { - editorOutput = safeExpandLiteralStrings(value as string); - } else if (isMapboxVectorTile(contentType)) { - const vectorTile = new VectorTile(new Protobuf(value as ArrayBuffer)); - const vectorTileJson = convertMapboxVectorTileToJson(vectorTile); - editorOutput = safeExpandLiteralStrings(vectorTileJson as string); - } else { - editorOutput = value; - } - - return editorOutput; - }) - .join('\n'), - mode - ); - } else if (error) { - const mode = modeForContentType(error.response.contentType); - editor.update(error.response.value as string, mode); - } else { - editor.update(''); - } - }, [readOnlySettings, data, error]); - - useEffect(() => { - applyCurrentSettings(editorInstanceRef.current!, readOnlySettings); - }, [readOnlySettings]); - - return ( - <> - - - -
-
-
- - ); -} - -export const EditorOutput = React.memo(EditorOutputUI); diff --git a/src/plugins/console/public/application/containers/editor/legacy/console_editor/keyboard_shortcuts.ts b/src/plugins/console/public/application/containers/editor/legacy/console_editor/keyboard_shortcuts.ts deleted file mode 100644 index daad4bbdb7dbd..0000000000000 --- a/src/plugins/console/public/application/containers/editor/legacy/console_editor/keyboard_shortcuts.ts +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import { throttle } from 'lodash'; -import { SenseEditor } from '../../../../models/sense_editor'; - -interface Actions { - senseEditor: SenseEditor; - sendCurrentRequest: () => void; - openDocumentation: () => void; -} - -const COMMANDS = { - SEND_TO_ELASTICSEARCH: 'send to Elasticsearch', - OPEN_DOCUMENTATION: 'open documentation', - AUTO_INDENT_REQUEST: 'auto indent request', - MOVE_TO_PREVIOUS_REQUEST: 'move to previous request start or end', - MOVE_TO_NEXT_REQUEST: 'move to next request start or end', - GO_TO_LINE: 'gotoline', -}; - -export function registerCommands({ senseEditor, sendCurrentRequest, openDocumentation }: Actions) { - const throttledAutoIndent = throttle(() => senseEditor.autoIndent(), 500, { - leading: true, - trailing: true, - }); - const coreEditor = senseEditor.getCoreEditor(); - - coreEditor.registerKeyboardShortcut({ - keys: { win: 'Ctrl-Enter', mac: 'Command-Enter' }, - name: COMMANDS.SEND_TO_ELASTICSEARCH, - fn: () => { - sendCurrentRequest(); - }, - }); - - coreEditor.registerKeyboardShortcut({ - name: COMMANDS.OPEN_DOCUMENTATION, - keys: { win: 'Ctrl-/', mac: 'Command-/' }, - fn: () => { - openDocumentation(); - }, - }); - - coreEditor.registerKeyboardShortcut({ - name: COMMANDS.AUTO_INDENT_REQUEST, - keys: { win: 'Ctrl-I', mac: 'Command-I' }, - fn: () => { - throttledAutoIndent(); - }, - }); - - coreEditor.registerKeyboardShortcut({ - name: COMMANDS.MOVE_TO_PREVIOUS_REQUEST, - keys: { win: 'Ctrl-Up', mac: 'Command-Up' }, - fn: () => { - senseEditor.moveToPreviousRequestEdge(); - }, - }); - - coreEditor.registerKeyboardShortcut({ - name: COMMANDS.MOVE_TO_NEXT_REQUEST, - keys: { win: 'Ctrl-Down', mac: 'Command-Down' }, - fn: () => { - senseEditor.moveToNextRequestEdge(false); - }, - }); - - coreEditor.registerKeyboardShortcut({ - name: COMMANDS.GO_TO_LINE, - keys: { win: 'Ctrl-L', mac: 'Command-L' }, - fn: (editor) => { - const line = parseInt(prompt('Enter line number') ?? '', 10); - if (!isNaN(line)) { - editor.gotoLine(line); - } - }, - }); -} - -export function unregisterCommands(senseEditor: SenseEditor) { - const coreEditor = senseEditor.getCoreEditor(); - Object.values(COMMANDS).forEach((command) => { - coreEditor.unregisterKeyboardShortcut(command); - }); -} diff --git a/src/plugins/console/public/application/containers/editor/legacy/console_menu_actions.ts b/src/plugins/console/public/application/containers/editor/legacy/console_menu_actions.ts deleted file mode 100644 index c65efbc0d82f5..0000000000000 --- a/src/plugins/console/public/application/containers/editor/legacy/console_menu_actions.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import { getEndpointFromPosition } from '../../../../lib/autocomplete/get_endpoint_from_position'; -import { SenseEditor } from '../../../models/sense_editor'; - -export async function autoIndent(editor: SenseEditor, event: React.MouseEvent) { - event.preventDefault(); - await editor.autoIndent(); - editor.getCoreEditor().getContainer().focus(); -} - -export function getDocumentation( - editor: SenseEditor, - docLinkVersion: string -): Promise { - return editor.getRequestsInRange().then((requests) => { - if (!requests || requests.length === 0) { - return null; - } - const position = requests[0].range.end; - position.column = position.column - 1; - const endpoint = getEndpointFromPosition(editor.getCoreEditor(), position, editor.parser); - if (endpoint && endpoint.documentation && endpoint.documentation.indexOf('http') !== -1) { - return endpoint.documentation - .replace('/master/', `/${docLinkVersion}/`) - .replace('/current/', `/${docLinkVersion}/`) - .replace('/{branch}/', `/${docLinkVersion}/`); - } else { - return null; - } - }); -} diff --git a/src/plugins/console/public/application/containers/editor/legacy/index.ts b/src/plugins/console/public/application/containers/editor/legacy/index.ts deleted file mode 100644 index 40e74e7b32e9e..0000000000000 --- a/src/plugins/console/public/application/containers/editor/legacy/index.ts +++ /dev/null @@ -1,11 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -export { EditorOutput, Editor } from './console_editor'; -export { getDocumentation, autoIndent } from './console_menu_actions'; diff --git a/src/plugins/console/public/application/containers/editor/legacy/subscribe_console_resize_checker.ts b/src/plugins/console/public/application/containers/editor/legacy/subscribe_console_resize_checker.ts deleted file mode 100644 index 6511d7ad3cc3b..0000000000000 --- a/src/plugins/console/public/application/containers/editor/legacy/subscribe_console_resize_checker.ts +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import { ResizeChecker } from '@kbn/kibana-utils-plugin/public'; - -export function subscribeResizeChecker(el: HTMLElement, ...editors: any[]) { - const checker = new ResizeChecker(el); - checker.on('resize', () => - editors.forEach((e) => { - if (e.getCoreEditor) { - e.getCoreEditor().resize(); - } else { - e.resize(); - } - - if (e.updateActionsBar) { - e.updateActionsBar(); - } - }) - ); - return () => checker.destroy(); -} diff --git a/src/plugins/console/public/application/containers/editor/monaco/index.ts b/src/plugins/console/public/application/containers/editor/monaco/index.ts deleted file mode 100644 index b7b8576bbdf65..0000000000000 --- a/src/plugins/console/public/application/containers/editor/monaco/index.ts +++ /dev/null @@ -1,11 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -export { MonacoEditor } from './monaco_editor'; -export { MonacoEditorOutput } from './monaco_editor_output'; diff --git a/src/plugins/console/public/application/containers/editor/monaco/monaco_editor.tsx b/src/plugins/console/public/application/containers/editor/monaco_editor.tsx similarity index 79% rename from src/plugins/console/public/application/containers/editor/monaco/monaco_editor.tsx rename to src/plugins/console/public/application/containers/editor/monaco_editor.tsx index ca6e66a8ba66f..bc174b772bb1c 100644 --- a/src/plugins/console/public/application/containers/editor/monaco/monaco_editor.tsx +++ b/src/plugins/console/public/application/containers/editor/monaco_editor.tsx @@ -8,18 +8,19 @@ */ import React, { CSSProperties, useCallback, useMemo, useRef, useState, useEffect } from 'react'; -import { EuiFlexGroup, EuiFlexItem, EuiIcon, EuiLink, EuiToolTip } from '@elastic/eui'; +import { EuiFlexGroup, EuiFlexItem, EuiButtonIcon, EuiToolTip } from '@elastic/eui'; import { css } from '@emotion/react'; import { CodeEditor } from '@kbn/code-editor'; import { CONSOLE_LANG_ID, CONSOLE_THEME_ID, monaco } from '@kbn/monaco'; import { i18n } from '@kbn/i18n'; -import { useSetInputEditor } from '../../../hooks'; +import { useSetInputEditor } from '../../hooks'; import { ContextMenu } from './components'; import { useServicesContext, useEditorReadContext, useRequestActionContext, -} from '../../../contexts'; + useEditorActionContext, +} from '../../contexts'; import { useSetInitialValue, useSetupAutocompletePolling, @@ -32,10 +33,12 @@ import { MonacoEditorActionsProvider } from './monaco_editor_actions_provider'; import { getSuggestionProvider } from './monaco_editor_suggestion_provider'; export interface EditorProps { - initialTextValue: string; + localStorageValue: string | undefined; + value: string; + setValue: (value: string) => void; } -export const MonacoEditor = ({ initialTextValue }: EditorProps) => { +export const MonacoEditor = ({ localStorageValue, value, setValue }: EditorProps) => { const context = useServicesContext(); const { services: { notifications, settings: settingsService, autocompleteInfo }, @@ -43,7 +46,11 @@ export const MonacoEditor = ({ initialTextValue }: EditorProps) => { config: { isDevMode }, } = context; const { toasts } = notifications; - const { settings } = useEditorReadContext(); + const { + settings, + restoreRequestFromHistory: requestToRestoreFromHistory, + fileToImport, + } = useEditorReadContext(); const [editorInstance, setEditorInstace] = useState< monaco.editor.IStandaloneCodeEditor | undefined >(); @@ -53,6 +60,7 @@ export const MonacoEditor = ({ initialTextValue }: EditorProps) => { const { registerKeyboardCommands, unregisterKeyboardCommands } = useKeyboardCommandsUtils(); const dispatch = useRequestActionContext(); + const editorDispatch = useEditorActionContext(); const actionsProvider = useRef(null); const [editorActionsCss, setEditorActionsCss] = useState({}); @@ -117,18 +125,40 @@ export const MonacoEditor = ({ initialTextValue }: EditorProps) => { const suggestionProvider = useMemo(() => { return getSuggestionProvider(actionsProvider); }, []); - const [value, setValue] = useState(initialTextValue); - useSetInitialValue({ initialTextValue, setValue, toasts }); + useSetInitialValue({ localStorageValue, setValue, toasts }); useSetupAutocompletePolling({ autocompleteInfo, settingsService }); useSetupAutosave({ value }); + // Restore the request from history if there is one + const updateEditor = useCallback(async () => { + if (requestToRestoreFromHistory) { + editorDispatch({ type: 'clearRequestToRestore' }); + await actionsProvider.current?.appendRequestToEditor( + requestToRestoreFromHistory, + dispatch, + context + ); + } + + // Import a request file if one is provided + if (fileToImport) { + editorDispatch({ type: 'setFileToImport', payload: null }); + await actionsProvider.current?.importRequestsToEditor(fileToImport); + } + }, [fileToImport, requestToRestoreFromHistory, dispatch, context, editorDispatch]); + + useEffect(() => { + updateEditor(); + }, [updateEditor]); + return (
{ - + - - - + iconSize={'s'} + /> - + { }; }); -jest.mock('../../../../services', () => { +jest.mock('../../../services', () => { return { getStorage: () => ({ get: () => [], @@ -40,7 +40,7 @@ jest.mock('../../../../services', () => { }; }); -jest.mock('../../../../lib/autocomplete/engine', () => { +jest.mock('../../../lib/autocomplete/engine', () => { return { populateContext: (...args: any) => { mockPopulateContext(args); diff --git a/src/plugins/console/public/application/containers/editor/monaco/monaco_editor_actions_provider.ts b/src/plugins/console/public/application/containers/editor/monaco_editor_actions_provider.ts similarity index 86% rename from src/plugins/console/public/application/containers/editor/monaco/monaco_editor_actions_provider.ts rename to src/plugins/console/public/application/containers/editor/monaco_editor_actions_provider.ts index 14be049c8ab26..8c66d31b2b57e 100644 --- a/src/plugins/console/public/application/containers/editor/monaco/monaco_editor_actions_provider.ts +++ b/src/plugins/console/public/application/containers/editor/monaco_editor_actions_provider.ts @@ -13,11 +13,11 @@ import { ConsoleParsedRequestsProvider, getParsedRequestsProvider, monaco } from import { i18n } from '@kbn/i18n'; import { toMountPoint } from '@kbn/react-kibana-mount'; import { XJson } from '@kbn/es-ui-shared-plugin/public'; -import { isQuotaExceededError } from '../../../../services/history'; -import { DEFAULT_VARIABLES } from '../../../../../common/constants'; -import { getStorage, StorageKeys } from '../../../../services'; -import { sendRequest } from '../../../hooks'; -import { Actions } from '../../../stores/request'; +import { isQuotaExceededError } from '../../../services/history'; +import { DEFAULT_VARIABLES } from '../../../../common/constants'; +import { getStorage, StorageKeys } from '../../../services'; +import { sendRequest } from '../../hooks'; +import { Actions } from '../../stores/request'; import { AutocompleteType, @@ -40,8 +40,9 @@ import { } from './utils'; import type { AdjustedParsedRequest } from './types'; -import { StorageQuotaError } from '../../../components/storage_quota_error'; -import { ContextValue } from '../../../contexts'; +import { type RequestToRestore, RestoreMethod } from '../../../types'; +import { StorageQuotaError } from '../../components/storage_quota_error'; +import { ContextValue } from '../../contexts'; import { containsComments, indentData } from './utils/requests_utils'; const AUTO_INDENTATION_ACTION_LABEL = 'Apply indentations'; @@ -120,7 +121,8 @@ export class MonacoEditorActionsProvider { const offset = this.editor.getTopForLineNumber(lineNumber) - this.editor.getScrollTop(); this.setEditorActionsCss({ visibility: 'visible', - top: offset, + // Move position down by 1 px so that the action buttons panel doesn't cover the top border of the selected block + top: offset + 1, }); } } @@ -147,7 +149,7 @@ export class MonacoEditorActionsProvider { range: selectedRange, options: { isWholeLine: true, - className: SELECTED_REQUESTS_CLASSNAME, + blockClassName: SELECTED_REQUESTS_CLASSNAME, }, }, ]); @@ -160,6 +162,11 @@ export class MonacoEditorActionsProvider { private async getSelectedParsedRequests(): Promise { const model = this.editor.getModel(); + + if (!model) { + return []; + } + const selection = this.editor.getSelection(); if (!model || !selection) { return Promise.resolve([]); @@ -173,6 +180,9 @@ export class MonacoEditorActionsProvider { startLineNumber: number, endLineNumber: number ): Promise { + if (!model) { + return []; + } const parsedRequests = await this.parsedRequestsProvider.getRequests(); const selectedRequests: AdjustedParsedRequest[] = []; for (const [index, parsedRequest] of parsedRequests.entries()) { @@ -243,9 +253,17 @@ export class MonacoEditorActionsProvider { const { toasts } = notifications; try { const allRequests = await this.getRequests(); - // if any request doesnt have a method then we gonna treat it as a non-valid - // request - const requests = allRequests.filter((request) => request.method); + const selectedRequests = await this.getSelectedParsedRequests(); + + const requests = allRequests + // if any request doesnt have a method then we gonna treat it as a non-valid + // request + .filter((request) => request.method) + // map the requests to the original line number + .map((request, index) => ({ + ...request, + lineNumber: selectedRequests[index].startLineNumber, + })); // If we do have requests but none have methods we are not sending the request if (allRequests.length > 0 && !requests.length) { @@ -479,9 +497,6 @@ export class MonacoEditorActionsProvider { return this.getSuggestions(model, position, context); } - /* - * This function inserts a request from the history into the editor - */ public async restoreRequestFromHistory(request: string) { const model = this.editor.getModel(); if (!model) { @@ -679,4 +694,82 @@ export class MonacoEditorActionsProvider { this.editor.trigger(TRIGGER_SUGGESTIONS_ACTION_LABEL, TRIGGER_SUGGESTIONS_HANDLER_ID, {}); } } + + /* + * This function cleares out the editor content and replaces it with the provided requests + */ + public async importRequestsToEditor(requestsToImport: string) { + const model = this.editor.getModel(); + + if (!model) { + return; + } + + const edit: monaco.editor.IIdentifiedSingleEditOperation = { + range: model.getFullModelRange(), + text: requestsToImport, + forceMoveMarkers: true, + }; + + this.editor.executeEdits('restoreFromHistory', [edit]); + } + + /* + * This function inserts a request after the last request in the editor + */ + public async appendRequestToEditor( + req: RequestToRestore, + dispatch: Dispatch, + context: ContextValue + ) { + const model = this.editor.getModel(); + + if (!model) { + return; + } + + // 1 - Create an edit operation to insert the request after the last request + const lastLineNumber = model.getLineCount(); + const column = model.getLineMaxColumn(lastLineNumber); + const edit: monaco.editor.IIdentifiedSingleEditOperation = { + range: { + startLineNumber: lastLineNumber, + startColumn: column, + endLineNumber: lastLineNumber, + endColumn: column, + }, + text: `\n\n${req.request}`, + forceMoveMarkers: true, + }; + this.editor.executeEdits('restoreFromHistory', [edit]); + + // 2 - Since we add two new lines, the cursor should be at the beginning of the new request + const beginningOfNewReq = lastLineNumber + 2; + const selectedRequests = await this.getRequestsBetweenLines( + model, + beginningOfNewReq, + beginningOfNewReq + ); + // We can assume that there is only one request given that we only add one + // request at a time. + const restoredRequest = selectedRequests[0]; + + // 3 - Set the cursor to the beginning of the new request, + this.editor.setSelection({ + startLineNumber: restoredRequest.startLineNumber, + startColumn: 1, + endLineNumber: restoredRequest.startLineNumber, + endColumn: 1, + }); + + // 4 - Scroll to the beginning of the new request + this.editor.setScrollPosition({ + scrollTop: this.editor.getTopForLineNumber(restoredRequest.startLineNumber), + }); + + // 5 - Optionally send the request + if (req.restoreMethod === RestoreMethod.RESTORE_AND_EXECUTE) { + this.sendRequests(dispatch, context); + } + } } diff --git a/src/plugins/console/public/application/containers/editor/monaco/monaco_editor_output.tsx b/src/plugins/console/public/application/containers/editor/monaco_editor_output.tsx similarity index 60% rename from src/plugins/console/public/application/containers/editor/monaco/monaco_editor_output.tsx rename to src/plugins/console/public/application/containers/editor/monaco_editor_output.tsx index 9de6748b62b6c..b9e3f3e6f9885 100644 --- a/src/plugins/console/public/application/containers/editor/monaco/monaco_editor_output.tsx +++ b/src/plugins/console/public/application/containers/editor/monaco_editor_output.tsx @@ -7,26 +7,44 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import React, { FunctionComponent, useCallback, useEffect, useRef, useState } from 'react'; +import React, { + CSSProperties, + FunctionComponent, + useCallback, + useEffect, + useRef, + useState, +} from 'react'; import { CodeEditor } from '@kbn/code-editor'; import { css } from '@emotion/react'; import { VectorTile } from '@mapbox/vector-tile'; import Protobuf from 'pbf'; import { i18n } from '@kbn/i18n'; -import { EuiScreenReaderOnly } from '@elastic/eui'; +import { + EuiScreenReaderOnly, + EuiFlexGroup, + EuiFlexItem, + EuiButtonIcon, + EuiToolTip, +} from '@elastic/eui'; import { CONSOLE_THEME_ID, CONSOLE_OUTPUT_LANG_ID, monaco } from '@kbn/monaco'; -import { getStatusCodeDecorations } from './utils'; -import { useEditorReadContext, useRequestReadContext } from '../../../contexts'; -import { convertMapboxVectorTileToJson } from '../legacy/console_editor/mapbox_vector_tile'; import { + getStatusCodeDecorations, isJSONContentType, isMapboxVectorTile, safeExpandLiteralStrings, languageForContentType, -} from '../utilities'; + convertMapboxVectorTileToJson, +} from './utils'; +import { useEditorReadContext, useRequestReadContext, useServicesContext } from '../../contexts'; +import { MonacoEditorOutputActionsProvider } from './monaco_editor_output_actions_provider'; import { useResizeCheckerUtils } from './hooks'; export const MonacoEditorOutput: FunctionComponent = () => { + const context = useServicesContext(); + const { + services: { notifications }, + } = context; const { settings: readOnlySettings } = useEditorReadContext(); const { lastResult: { data }, @@ -37,8 +55,14 @@ export const MonacoEditorOutput: FunctionComponent = () => { const { setupResizeChecker, destroyResizeChecker } = useResizeCheckerUtils(); const lineDecorations = useRef(null); + const actionsProvider = useRef(null); + const [editorActionsCss, setEditorActionsCss] = useState({}); + const editorDidMountCallback = useCallback( (editor: monaco.editor.IStandaloneCodeEditor) => { + const provider = new MonacoEditorOutputActionsProvider(editor, setEditorActionsCss); + actionsProvider.current = provider; + setupResizeChecker(divRef.current!, editor); lineDecorations.current = editor.createDecorationsCollection(); }, @@ -83,19 +107,71 @@ export const MonacoEditorOutput: FunctionComponent = () => { // If there are multiple responses, add decorations for their status codes const decorations = getStatusCodeDecorations(data); lineDecorations.current?.set(decorations); + // Highlight first line of the output editor + actionsProvider.current?.selectFirstLine(); } } else { setValue(''); } }, [readOnlySettings, data, value]); + const copyOutputCallback = useCallback(async () => { + const selectedText = (await actionsProvider.current?.getParsedOutput()) as string; + + try { + if (!window.navigator?.clipboard) { + throw new Error('Could not copy to clipboard!'); + } + + await window.navigator.clipboard.writeText(selectedText); + + notifications.toasts.addSuccess({ + title: i18n.translate('console.outputPanel.copyOutputToast', { + defaultMessage: 'Selected output copied to clipboard', + }), + }); + } catch (e) { + notifications.toasts.addDanger({ + title: i18n.translate('console.outputPanel.copyOutputToastFailedMessage', { + defaultMessage: 'Could not copy selected output to clipboard', + }), + }); + } + }, [notifications.toasts]); + return (
+ + + + + + +