[ML] Add control to show or hide empty fields in dropdown in Transform

x-pack/packages/ml/field_stats_flyout/options_list_with_stats/option_list_popover.tsx

@@ -107,7 +107,9 @@ export const OptionsListPopover = ({
 }: OptionsListPopoverProps) => {
   const { populatedFields } = useFieldStatsFlyoutContext();
 
-  const [showEmptyFields, setShowEmptyFields] = useState(false);
+  const [showEmptyFields, setShowEmptyFields] = useState(
+    populatedFields ? !(populatedFields.size > 0) : true
+  );
   const id = useMemo(() => htmlIdGenerator()(), []);
 
   const filteredOptions = useMemo(() => {

x-pack/packages/ml/field_stats_flyout/options_list_with_stats/option_list_with_stats.tsx

@@ -7,7 +7,11 @@
 
 import type { FC } from 'react';
 import React, { useMemo, useState } from 'react';
-import type { EuiComboBoxOptionOption, EuiComboBoxSingleSelectionShape } from '@elastic/eui';
+import type {
+  EuiComboBoxOptionOption,
+  EuiComboBoxSingleSelectionShape,
+  EuiFormControlLayoutProps,
+} from '@elastic/eui';
 import { EuiInputPopover, htmlIdGenerator, EuiFormControlLayout, EuiFieldText } from '@elastic/eui';
 import { css } from '@emotion/react';
 import { i18n } from '@kbn/i18n';
@@ -18,8 +22,6 @@ import type { DropDownLabel } from './types';
 const MIN_POPOVER_WIDTH = 400;
 
 export const optionCss = css`
-  display: flex;
-  align-items: center;
   .euiComboBoxOption__enterBadge {
     display: none;
   }
@@ -31,7 +33,8 @@ export const optionCss = css`
   }
 `;
 
-interface OptionListWithFieldStatsProps {
+interface OptionListWithFieldStatsProps
+  extends Pick<EuiFormControlLayoutProps, 'prepend' | 'compressed'> {
   options: DropDownLabel[];
   placeholder?: string;
   'aria-label'?: string;
@@ -58,6 +61,8 @@ export const OptionListWithFieldStats: FC<OptionListWithFieldStatsProps> = ({
   isDisabled,
   isLoading,
   isClearable = true,
+  prepend,
+  compressed,
   'aria-label': ariaLabel,
   'data-test-subj': dataTestSubj,
 }) => {
@@ -68,13 +73,12 @@ export const OptionListWithFieldStats: FC<OptionListWithFieldStatsProps> = ({
   const comboBoxOptions: DropDownLabel[] = useMemo(
     () =>
       Array.isArray(options)
-        ? options.map(({ isEmpty, hideTrigger: hideInspectButton, ...o }) => ({
+        ? options.map(({ isEmpty, ...o }) => ({
             ...o,
             css: optionCss,
             // Change data-is-empty- because EUI is passing all props to dom element
             // so isEmpty is invalid, but we need this info to render option correctly
             'data-is-empty': isEmpty,
-            'data-hide-inspect': hideInspectButton,
           }))
         : [],
     [options]
@@ -89,6 +93,8 @@ export const OptionListWithFieldStats: FC<OptionListWithFieldStatsProps> = ({
       id={popoverId}
       input={
         <EuiFormControlLayout
+          prepend={prepend}
+          compressed={compressed}
           fullWidth={fullWidth}
           // Adding classname to make functional tests similar to EuiComboBox
           className={singleSelection ? 'euiComboBox__inputWrap--plainText' : ''}

x-pack/packages/ml/field_stats_flyout/options_list_with_stats/types.ts

@@ -9,8 +9,9 @@ import type { EuiComboBoxOptionOption, EuiSelectableOption } from '@elastic/eui'
 import type { Aggregation, Field } from '@kbn/ml-anomaly-utils';
 
 interface BaseOption<T> {
-  key?: string;
   label: string | React.ReactNode;
+  key?: string;
+  value?: string | number | string[];
   isEmpty?: boolean;
   hideTrigger?: boolean;
   'data-is-empty'?: boolean;

x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_dropdown/dropdown.tsx

@@ -8,7 +8,7 @@
 import React from 'react';
 
 import type { EuiComboBoxOptionsListProps, EuiComboBoxOptionOption } from '@elastic/eui';
-import { EuiComboBox } from '@elastic/eui';
+import { OptionListWithFieldStats } from '@kbn/ml-field-stats-flyout/options_list_with_stats/option_list_with_stats';
 
 interface Props {
   options: EuiComboBoxOptionOption[];
@@ -30,7 +30,7 @@ export const DropDown: React.FC<Props> = ({
   isDisabled,
 }) => {
   return (
-    <EuiComboBox
+    <OptionListWithFieldStats
       fullWidth
       placeholder={placeholder}
       singleSelection={{ asPlainText: true }}
@@ -40,7 +40,6 @@ export const DropDown: React.FC<Props> = ({
       isClearable={false}
       data-test-subj={testSubj}
       isDisabled={isDisabled}
-      renderOption={renderOption}
     />
   );
 };

x-pack/plugins/transform/public/app/sections/create_transform/components/aggregation_list/sub_aggs_section.tsx

@@ -11,11 +11,14 @@ import type { EuiComboBoxOptionOption } from '@elastic/eui';
 import { EuiSpacer, EuiToolTip } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n-react';
+import { FieldStatsInfoButton, useFieldStatsTrigger } from '@kbn/ml-field-stats-flyout';
 import { AggListForm } from './list_form';
 import { DropDown } from '../aggregation_dropdown';
 import type { PivotAggsConfig } from '../../../../common';
 import { PivotConfigurationContext } from '../pivot_configuration/pivot_configuration';
 import { MAX_NESTING_SUB_AGGS } from '../../../../common/pivot_aggs';
+import type { DropDownOptionWithField } from '../step_define/common/get_pivot_dropdown_options';
+import type { DropDownOption } from '../../../../common/dropdown';
 
 /**
  * Component for managing sub-aggregation of the provided
@@ -54,11 +57,47 @@ export const SubAggsSection: FC<{ item: PivotAggsConfig }> = ({ item }) => {
     }
     return nestingLevel <= MAX_NESTING_SUB_AGGS;
   }, [item]);
+  const { handleFieldStatsButtonClick, populatedFields } = useFieldStatsTrigger();
 
+  const options = useMemo(() => {
+    const opts: EuiComboBoxOptionOption[] = [];
+    state.aggOptions.forEach(({ label, field, options: aggOptions }: DropDownOptionWithField) => {
+      const isEmpty = populatedFields && field.id ? !populatedFields.has(field.id) : false;
+
+      const aggOption: DropDownOption = {
+        isGroupLabel: true,
+        key: field.id,
+        searchableLabel: label,
+        // @ts-ignore Purposefully passing label as element instead of string
+        // for more robust rendering
+        label: (
+          <FieldStatsInfoButton
+            isEmpty={populatedFields && !populatedFields.has(field.id)}
+            field={field}
+            label={label}
+            onButtonClick={handleFieldStatsButtonClick}
+          />
+        ),
+      };
+
+      if (aggOptions.length) {
+        opts.push(aggOption);
+        opts.push(
+          ...aggOptions.map((o) => ({
+            ...o,
+            isEmpty,
+            isGroupLabel: false,
+            searchableLabel: o.label,
+          }))
+        );
+      }
+    });
+    return opts;
+  }, [handleFieldStatsButtonClick, populatedFields, state.aggOptions]);
   const dropdown = (
     <DropDown
       changeHandler={addSubAggHandler}
-      options={state.aggOptions}
+      options={options}
       placeholder={i18n.translate('xpack.transform.stepDefineForm.addSubAggregationPlaceholder', {
         defaultMessage: 'Add a sub-aggregation ...',
       })}

x-pack/plugins/transform/public/app/sections/create_transform/components/pivot_configuration/pivot_configuration.tsx

@@ -12,7 +12,6 @@ import { EuiFormRow, type EuiComboBoxOptionOption } from '@elastic/eui';
 
 import { i18n } from '@kbn/i18n';
 import { useFieldStatsTrigger, FieldStatsInfoButton } from '@kbn/ml-field-stats-flyout';
-
 import { type DropDownOptionWithField } from '../step_define/common/get_pivot_dropdown_options';
 import type { DropDownOption } from '../../../../common';
 import { AggListForm } from '../aggregation_list';
@@ -41,28 +40,42 @@ export const PivotConfiguration: FC<StepDefineFormHook['pivotConfig']> = memo(
     const { aggList, aggOptions, aggOptionsData, groupByList, groupByOptions, groupByOptionsData } =
       state;
 
-    const aggOptionsWithFieldStats: EuiComboBoxOptionOption[] = useMemo(
-      () =>
-        aggOptions.map(({ label, field, options }: DropDownOptionWithField) => {
-          const aggOption: DropDownOption = {
-            isGroupLabelOption: true,
-            key: field.id,
-            // @ts-ignore Purposefully passing label as element instead of string
-            // for more robust rendering
-            label: (
-              <FieldStatsInfoButton
-                isEmpty={populatedFields && !populatedFields.has(field.id)}
-                field={field}
-                label={label}
-                onButtonClick={handleFieldStatsButtonClick}
-              />
-            ),
-            options: options ?? [],
-          };
-          return aggOption;
-        }),
-      [aggOptions, handleFieldStatsButtonClick, populatedFields]
-    );
+    const aggOptionsWithFieldStats: EuiComboBoxOptionOption[] = useMemo(() => {
+      const opts: EuiComboBoxOptionOption[] = [];
+      aggOptions.forEach(({ label, field, options }: DropDownOptionWithField) => {
+        const isEmpty = populatedFields && field.id ? !populatedFields.has(field.id) : false;
+
+        const aggOption: DropDownOption = {
+          isGroupLabel: true,
+          key: field.id,
+          searchableLabel: label,
+          // @ts-ignore Purposefully passing label as element instead of string
+          // for more robust rendering
+          label: (
+            <FieldStatsInfoButton
+              isEmpty={populatedFields && !populatedFields.has(field.id)}
+              field={field}
+              label={label}
+              onButtonClick={handleFieldStatsButtonClick}
+            />
+          ),
+        };
+
+        if (options.length) {
+          opts.push(aggOption);
+          opts.push(
+            ...options.map((o) => ({
+              ...o,
+              isEmpty,
+              isGroupLabel: false,
+              searchableLabel: o.label,
+            }))
+          );
+        }
+      });
+      return opts;
+    }, [aggOptions, handleFieldStatsButtonClick, populatedFields]);
+
     return (
       <PivotConfigurationContext.Provider value={{ actions, state }}>
         <EuiFormRow

x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/latest_function_form.tsx

@@ -10,7 +10,8 @@ import React, { type FC } from 'react';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n-react';
 import { EuiButtonIcon, EuiCallOut, EuiComboBox, EuiCopy, EuiFormRow } from '@elastic/eui';
-import { useFieldStatsTrigger } from '@kbn/ml-field-stats-flyout';
+import type { DropDownLabel } from '@kbn/ml-field-stats-flyout';
+import { OptionListWithFieldStats, useFieldStatsTrigger } from '@kbn/ml-field-stats-flyout';
 import type { LatestFunctionService } from './hooks/use_latest_function_config';
 
 interface LatestFunctionFormProps {
@@ -73,7 +74,7 @@ export const LatestFunctionForm: FC<LatestFunctionFormProps> = ({
       >
         <>
           {latestFunctionService.sortFieldOptions.length > 0 && (
-            <EuiComboBox
+            <OptionListWithFieldStats
               fullWidth
               placeholder={i18n.translate('xpack.transform.stepDefineForm.sortPlaceholder', {
                 defaultMessage: 'Add a date field ...',
@@ -83,15 +84,19 @@ export const LatestFunctionForm: FC<LatestFunctionFormProps> = ({
               selectedOptions={
                 latestFunctionService.config.sort ? [latestFunctionService.config.sort] : []
               }
-              onChange={(selected) => {
-                latestFunctionService.updateLatestFunctionConfig({
-                  sort: { value: selected[0].value, label: selected[0].label as string },
-                });
+              onChange={(selected: DropDownLabel[]) => {
+                if (typeof selected[0].value === 'string') {
+                  latestFunctionService.updateLatestFunctionConfig({
+                    sort: {
+                      value: selected[0].value,
+                      label: selected[0].label?.toString(),
+                    },
+                  });
+                }
                 closeFlyout();
               }}
               isClearable={false}
               data-test-subj="transformWizardSortFieldSelector"
-              renderOption={renderOption}
             />
           )}
           {latestFunctionService.sortFieldOptions.length === 0 && (

x-pack/test/functional/services/transform/wizard.ts

@@ -444,7 +444,10 @@ export function TransformWizardProvider({ getService, getPageObjects }: FtrProvi
     },
 
     async setSortFieldValue(identificator: string, label: string) {
-      await comboBox.set('transformWizardSortFieldSelector > comboBoxInput', identificator);
+      await ml.commonUI.setOptionsListWithFieldStatsValue(
+        'transformWizardSortFieldSelector > comboBoxInput',
+        identificator
+      );
       await this.assertSortFieldInputValue(identificator);
     },
 
@@ -507,7 +510,10 @@ export function TransformWizardProvider({ getService, getPageObjects }: FtrProvi
       expectedLabel: string,
       expectedIntervalLabel?: string
     ) {
-      await comboBox.set('transformGroupBySelection > comboBoxInput', identifier);
+      await ml.commonUI.setOptionsListWithFieldStatsValue(
+        'transformGroupBySelection > comboBoxInput',
+        identifier
+      );
       await this.assertGroupByInputValue([]);
       await this.assertGroupByEntryExists(index, expectedLabel, expectedIntervalLabel);
     },
@@ -582,7 +588,10 @@ export function TransformWizardProvider({ getService, getPageObjects }: FtrProvi
       formData?: Record<string, any>,
       parentSelector = ''
     ) {
-      await comboBox.set(this.getAggComboBoxInputSelector(parentSelector), identifier);
+      await ml.commonUI.setOptionsListWithFieldStatsValue(
+        this.getAggComboBoxInputSelector(parentSelector),
+        identifier
+      );
       await this.assertAggregationInputValue([], parentSelector);
       await this.assertAggregationEntryExists(index, expectedLabel, parentSelector);