[Do not merge] - Dataview perf improvements poc

x-pack/solutions/security/plugins/security_solution/public/app/home/index.tsx

@@ -25,7 +25,7 @@ import { useUpgradeSecurityPackages } from '../../detection_engine/rule_manageme
 import { useSetupDetectionEngineHealthApi } from '../../detection_engine/rule_monitoring';
 import { TopValuesPopover } from '../components/top_values_popover/top_values_popover';
 import { AssistantOverlay } from '../../assistant/overlay';
-import { useInitSourcerer } from '../../sourcerer/containers/use_init_sourcerer';
+// import { useInitSourcerer } from '../../sourcerer/containers/use_init_sourcerer';
 import { useInitDataViewManager } from '../../data_view_manager/hooks/use_init_data_view_manager';
 import { useRestoreDataViewManagerStateFromURL } from '../../data_view_manager/hooks/use_sync_url_state';
 import { useBrowserFields } from '../../data_view_manager/hooks/use_browser_fields';
@@ -41,7 +41,7 @@ const HomePageComponent: React.FC<HomePageProps> = ({ children }) => {
 
   const { pathname } = useLocation();
   const sourcererScope = getScopeFromPath(pathname);
-  const { browserFields: oldBrowserFields } = useInitSourcerer(sourcererScope);
+  // const { browserFields: oldBrowserFields } = useInitSourcerer(sourcererScope);
   const { browserFields: experimentalBrowserFields } = useBrowserFields(sourcererScope);
 
   useRestoreDataViewManagerStateFromURL(useInitDataViewManager(), sourcererScope);
@@ -51,7 +51,7 @@ const HomePageComponent: React.FC<HomePageProps> = ({ children }) => {
   useUpdateExecutionContext();
 
   const browserFields = (
-    newDataViewPickerEnabled ? experimentalBrowserFields : oldBrowserFields
+    newDataViewPickerEnabled ? experimentalBrowserFields : {}
   ) as BrowserFields;
 
   // side effect: this will attempt to upgrade the endpoint package if it is not up to date

x-pack/solutions/security/plugins/security_solution/public/assistant/send_to_timeline/index.tsx

@@ -67,7 +67,7 @@ export const SendToTimelineButton: FC<PropsWithChildren<SendToTimelineButtonProp
   const { dataViewId: oldTimelineDataViewId } = useSourcererDataView(SourcererScopeName.timeline);
   const newDataViewPickerEnabled = useIsExperimentalFeatureEnabled('newDataViewPickerEnabled');
 
-  const { dataViewSpec } = useDataViewSpec(SourcererScopeName.timeline);
+  const { dataViewSpec } = useDataViewSpec(SourcererScopeName.timeline, false);
 
   const timelineDataViewId = newDataViewPickerEnabled
     ? dataViewSpec?.id ?? null

x-pack/solutions/security/plugins/security_solution/public/common/components/cell_actions/index.tsx

@@ -14,11 +14,13 @@ import type {
 import React, { useMemo } from 'react';
 import type { CellActionFieldValue, CellActionsData } from '@kbn/cell-actions/src/types';
 import type { EuiButtonIconProps } from '@elastic/eui';
+import { useDataViewSpec } from '../../../data_view_manager/hooks/use_data_view_spec';
 import type { SecurityCellActionMetadata } from '../../../app/actions/types';
 import { SecurityCellActionsTrigger, SecurityCellActionType } from '../../../app/actions/constants';
 import { SourcererScopeName } from '../../../sourcerer/store/model';
 import { useGetFieldSpec } from '../../hooks/use_get_field_spec';
 import { useDataViewId } from '../../hooks/use_data_view_id';
+import { useIsExperimentalFeatureEnabled } from '../../hooks/use_experimental_features';
 
 // bridge exports for convenience
 export * from '@kbn/cell-actions';
@@ -64,8 +66,14 @@ export const SecurityCellActions: React.FC<SecurityCellActionsProps> = ({
   children,
   ...props
 }) => {
+  const newDataViewPickerEnabled = useIsExperimentalFeatureEnabled('newDataViewPickerEnabled');
+  const { dataViewSpec } = useDataViewSpec(sourcererScopeId);
+  const experimentalDataViewId = dataViewSpec?.id ?? '';
+
   const getFieldSpec = useGetFieldSpec(sourcererScopeId);
-  const dataViewId = useDataViewId(sourcererScopeId);
+  const oldDataViewId = useDataViewId(sourcererScopeId);
+
+  const dataViewId = newDataViewPickerEnabled ? experimentalDataViewId : oldDataViewId;
   // Make a dependency key to prevent unnecessary re-renders when data object is defined inline
   // It is necessary because the data object is an array or an object and useMemo would always re-render
   const dependencyKey = JSON.stringify(data);
@@ -74,12 +82,12 @@ export const SecurityCellActions: React.FC<SecurityCellActionsProps> = ({
     () =>
       (Array.isArray(data) ? data : [data])
         .map(({ field, value }) => ({
-          field: getFieldSpec(field),
+          field: newDataViewPickerEnabled ? dataViewSpec.fields?.[field] : getFieldSpec(field),
           value,
         }))
         .filter((item): item is CellActionsData => !!item.field),
     // eslint-disable-next-line react-hooks/exhaustive-deps -- Use the dependencyKey to prevent unnecessary re-renders
-    [dependencyKey, getFieldSpec]
+    [dependencyKey, dataViewSpec, getFieldSpec, newDataViewPickerEnabled]
   );
 
   const metadataWithDataView = useMemo(() => ({ ...metadata, dataViewId }), [dataViewId, metadata]);

x-pack/solutions/security/plugins/security_solution/public/common/components/markdown_editor/plugins/insight/index.tsx

@@ -37,6 +37,7 @@ import { FormattedMessage } from '@kbn/i18n-react';
 import type { Filter } from '@kbn/es-query';
 import { FilterStateStore } from '@kbn/es-query';
 import { useForm, FormProvider, useController } from 'react-hook-form';
+import { DataViewManagerScopeName } from '../../../../../data_view_manager/constants';
 import { useDataView } from '../../../../../data_view_manager/hooks/use_data_view';
 import { useIsExperimentalFeatureEnabled } from '../../../../hooks/use_experimental_features';
 import { useUpsellingMessage } from '../../../../hooks/use_upselling';
@@ -291,7 +292,8 @@ const InsightEditorComponent = ({
 
   const newDataViewPickerEnabled = useIsExperimentalFeatureEnabled('newDataViewPickerEnabled');
 
-  const { dataViewSpec } = useDataViewSpec();
+  // Only need the name
+  const { dataViewSpec } = useDataViewSpec(DataViewManagerScopeName.default, false);
   const sourcererDataView = newDataViewPickerEnabled ? dataViewSpec : oldSourcererDataView;
 
   const {

x-pack/solutions/security/plugins/security_solution/public/common/containers/source/use_data_view.tsx

@@ -9,6 +9,7 @@ import { useCallback, useRef } from 'react';
 import type { Subscription } from 'rxjs';
 import { useDispatch } from 'react-redux';
 import memoizeOne from 'memoize-one';
+import deepEqual from 'fast-deep-equal';
 import type { BrowserFields } from '@kbn/timelines-plugin/common';
 import type { DataViewSpec } from '@kbn/data-views-plugin/public';
 import type { FieldCategory } from '@kbn/timelines-plugin/common/search_strategy';
@@ -66,9 +67,14 @@ export const getDataViewStateFromIndexFields = memoizeOne(
       return { browserFields: browserFields as DangerCastForBrowserFieldsMutation };
     }
   },
-  (newArgs, lastArgs) => newArgs[0] === lastArgs[0] && newArgs[1]?.length === lastArgs[1]?.length
+  (newArgs, lastArgs) => deepEqual(newArgs, lastArgs)
 );
 
+// This is a utility function to get an instance of the getDataViewStateFromIndexFields function
+// If the original function is called in a hook called in different places, the memoization becomes potential useless
+// as each call overrides the previous one. This hook is used to ensure that the memoization is preserved for each hook instance.
+export const getMemoizedGetDataViewStateFromIndexFields = () => getDataViewStateFromIndexFields;
+
 export const useDataView = (): {
   indexFieldsSearch: IndexFieldSearch;
 } => {

x-pack/solutions/security/plugins/security_solution/public/data_view_manager/components/data_view_picker/index.tsx

@@ -23,6 +23,7 @@ import { useManagedDataViews } from '../../hooks/use_managed_data_views';
 import { useSavedDataViews } from '../../hooks/use_saved_data_views';
 import { DEFAULT_SECURITY_DATA_VIEW, LOADING } from './translations';
 import { DATA_VIEW_PICKER_TEST_ID } from './constants';
+import { dataViewSpecCache } from '../../utils/data_view_spec_cache';
 
 interface DataViewPickerProps {
   /**
@@ -49,7 +50,7 @@ export const DataViewPicker = memo(({ scope, onClosePopover, disabled }: DataVie
   const closeDataViewEditor = useRef<() => void | undefined>();
   const closeFieldEditor = useRef<() => void | undefined>();
 
-  const { dataViewSpec, status } = useDataViewSpec(scope);
+  const { dataViewSpec, status } = useDataViewSpec(scope, false);
 
   const { adhocDataViews: adhocDataViewSpecs, defaultDataViewId } =
     useSelector(sharedStateSelector);
@@ -69,6 +70,8 @@ export const DataViewPicker = memo(({ scope, onClosePopover, disabled }: DataVie
   // hence - it is the only place where we should update the url param for the data view selection.
   const handleChangeDataView = useCallback(
     (id: string, indexPattern: string = '') => {
+      // Update the dataViewSpec cache when re-selecting it.
+      dataViewSpecCache.delete(id);
       selectDataView({ id, scope });
 
       if (isDefaultSourcerer) {
@@ -131,6 +134,7 @@ export const DataViewPicker = memo(({ scope, onClosePopover, disabled }: DataVie
       if (!updatedDataView.id) {
         return;
       }
+      dataViewSpecCache.delete(updatedDataView.id);
       handleChangeDataView(updatedDataView.id, updatedDataView.getIndexPattern());
     },
     [handleChangeDataView]

x-pack/solutions/security/plugins/security_solution/public/data_view_manager/hooks/use_browser_fields.ts

@@ -9,7 +9,7 @@ import { useMemo } from 'react';
 import type { BrowserFields } from '@kbn/timelines-plugin/common';
 import { DataViewManagerScopeName } from '../constants';
 import { useDataViewSpec } from './use_data_view_spec';
-import { getDataViewStateFromIndexFields } from '../../common/containers/source/use_data_view';
+import { getMemoizedGetDataViewStateFromIndexFields } from '../../common/containers/source/use_data_view';
 
 export const useBrowserFields = (
   scope: DataViewManagerScopeName = DataViewManagerScopeName.default
@@ -21,6 +21,7 @@ export const useBrowserFields = (
       return {};
     }
 
+    const getDataViewStateFromIndexFields = getMemoizedGetDataViewStateFromIndexFields();
     const { browserFields } = getDataViewStateFromIndexFields(
       dataViewSpec?.title ?? '',
       dataViewSpec.fields

x-pack/solutions/security/plugins/security_solution/public/data_view_manager/hooks/use_data_view_spec.ts

@@ -9,6 +9,7 @@ import { useMemo } from 'react';
 import type { DataViewSpec, SharedDataViewSelectionState } from '../redux/types';
 import { DataViewManagerScopeName } from '../constants';
 import { useDataView } from './use_data_view';
+import { dataViewSpecCache } from '../utils/data_view_spec_cache';
 
 export interface UseDataViewSpecResult {
   /**
@@ -25,14 +26,25 @@ export interface UseDataViewSpecResult {
  * Returns an object with the dataViewSpec and status values for the given scopeName.
  */
 export const useDataViewSpec = (
-  scopeName: DataViewManagerScopeName = DataViewManagerScopeName.default
+  scopeName: DataViewManagerScopeName = DataViewManagerScopeName.default,
+  // This can be prohibitively expensive with sufficient enough fields and called in enough components
+  includeFields: boolean = true
 ): UseDataViewSpecResult => {
   const { dataView, status } = useDataView(scopeName);
 
+  const cachedSpec = dataViewSpecCache.get(dataView?.id ?? '');
+  const shouldUpdateCacheWithFieldsInformation = cachedSpec && !cachedSpec.fields && includeFields;
+
+  if (dataView?.id && (!cachedSpec || shouldUpdateCacheWithFieldsInformation)) {
+    // Cache the DataViewSpec to avoid recalculating it every time the hook is called
+    dataViewSpecCache.set(dataView?.id, dataView.toSpec?.(includeFields));
+  }
+
   return useMemo(() => {
     // NOTE: remove this after we are ready for undefined (lazy) data view everywhere in the app
     // https://github.com/elastic/security-team/issues/11959
-    if (!dataView) {
+    // every dataView should have the saved object id
+    if (!dataView || !dataView.id) {
       return {
         dataViewSpec: {
           id: '',
@@ -41,7 +53,9 @@ export const useDataViewSpec = (
         status,
       };
     }
-
-    return { dataViewSpec: dataView?.toSpec?.(), status };
-  }, [dataView, status]);
+    const dataViewSpec = cachedSpec ?? dataView.toSpec?.(includeFields);
+    // TODO: (DV_PICKER) Remove this in the cleanup phase, just here for testing purposes
+    dataViewSpecCache.log();
+    return { dataViewSpec, status };
+  }, [cachedSpec, dataView, includeFields, status]);
 };

x-pack/solutions/security/plugins/security_solution/public/data_view_manager/hooks/use_init_data_view_manager.ts

@@ -21,6 +21,7 @@ import { sharedDataViewManagerSlice } from '../redux/slices';
 import { useUserInfo } from '../../detections/components/user_info';
 import { type SelectDataViewAsyncPayload } from '../redux/actions';
 import { DataViewManagerScopeName } from '../constants';
+import { dataViewSpecCache } from '../utils/data_view_spec_cache';
 
 type OriginalListener = Parameters<typeof originalAddListener>[0];
 
@@ -105,6 +106,7 @@ export const useInitDataViewManager = () => {
       listeners.forEach((dataViewSelectedListener) => {
         dispatch(removeListener(dataViewSelectedListener));
       });
+      dataViewSpecCache.clear();
     };
   }, [
     dispatch,

x-pack/solutions/security/plugins/security_solution/public/data_view_manager/utils/data_view_spec_cache.ts

@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { DataViewSpec } from '@kbn/data-views-plugin/common';
+
+/**
+ * @description We use a cache here rather than redux because the dataViewSpec is not
+ * serializable and we don't want to store it in the redux store.
+ * An alternative could be to store it in context of some sort at the top level of the application,
+ * but I would rather not mix context and redux for this use case.
+ */
+
+export class DataViewSpecCache {
+  // Ensure one cache is only ever created
+  static #instance: DataViewSpecCache | null = null;
+  private cache: Map<string, DataViewSpec> = new Map();
+
+  constructor() {
+    if (!DataViewSpecCache.#instance) {
+      DataViewSpecCache.#instance = this;
+    }
+    return DataViewSpecCache.#instance;
+  }
+
+  get(dataViewId: string): DataViewSpec | undefined {
+    return this.cache.get(dataViewId);
+  }
+
+  set(dataViewId: string, dataViewSpec: DataViewSpec): void {
+    this.cache.set(dataViewId, dataViewSpec);
+  }
+
+  clear(): void {
+    this.cache.clear();
+  }
+  /**
+   * Deletes the DataViewSpec from the cache by its ID.
+   * @param dataViewId - The ID of the DataViewSpec to delete.
+   */
+  delete(dataViewId: string): void {
+    this.cache.delete(dataViewId);
+  }
+
+  has(dataViewId: string): boolean {
+    return this.cache.has(dataViewId);
+  }
+
+  size(): number {
+    return this.cache.size;
+  }
+
+  // TODO: (DV_PICKER) Remove this in the cleanup phase, just here for testing purposes
+  log(): void {
+    console.debug('!!DataViewSpecCache contents:', Array.from(this.cache.entries()));
+  }
+}
+
+/**
+ * @description This cache is used to store the DataViewSpec objects to avoid recalculating them
+ * every time the useDataViewSpec hook is called. This is particularly useful
+ * when the dataView has a large number of fields, as the toSpec() method can be
+ * expensive to compute.
+ */
+export const dataViewSpecCache = new DataViewSpecCache();

x-pack/solutions/security/plugins/security_solution/public/detections/components/alerts_table/additional_toolbar_controls.tsx

@@ -80,6 +80,7 @@ const AdditionalToolbarControlsComponent = ({
     [dispatch, tableType, trackGroupChange]
   );
 
+  // TODO: (DV_PICKER) This can be generalized to just return the fields for the current data view.
   const fields = useMemo(() => {
     return Object.values(sourcererDataView.fields || {});
   }, [sourcererDataView.fields]);