[Cascade] Handle grouping on fields with unset values

src/platform/packages/shared/kbn-esql-utils/constants.ts

@@ -8,3 +8,8 @@
  */
 
 export const ENABLE_ESQL = 'enableESQL';
+
+/**
+ * Denotes placeholder value for property on a record that is not set.
+ */
+export const GROUP_NOT_SET_VALUE = '(null)';

src/platform/packages/shared/kbn-esql-utils/index.ts

@@ -73,4 +73,4 @@ export {
   type ESQLStatsQueryMeta,
 } from './src';
 
-export { ENABLE_ESQL } from './constants';
+export { ENABLE_ESQL, GROUP_NOT_SET_VALUE } from './constants';

src/platform/packages/shared/kbn-esql-utils/src/utils/cascaded_documents_helpers/index.test.ts

@@ -16,6 +16,7 @@ import {
   constructCascadeQuery,
   appendFilteringWhereClauseForCascadeLayout,
 } from '.';
+import { GROUP_NOT_SET_VALUE } from '../../../constants';
 
 describe('cascaded documents helpers utils', () => {
   const dataViewMock = createStubDataView({
@@ -466,6 +467,30 @@ describe('cascaded documents helpers utils', () => {
             'FROM kibana_sample_data_logs | WHERE MATCH_PHRASE(`tags.keyword`, "some random pattern")'
           );
         });
+
+        it('uses postfix unary expression when the selected column is the denoted "GROUP_NOT_SET_VALUE"', () => {
+          const editorQuery: AggregateQuery = {
+            esql: `
+              FROM kibana_sample_data_logs | STATS count = COUNT(bytes), average = AVG(memory) BY clientip
+            `,
+          };
+
+          const nodePath = ['clientip'];
+          const nodePathMap = { clientip: GROUP_NOT_SET_VALUE };
+
+          const cascadeQuery = constructCascadeQuery({
+            query: editorQuery,
+            dataView: dataViewMock,
+            esqlVariables: [],
+            nodeType,
+            nodePath,
+            nodePathMap,
+          });
+          expect(cascadeQuery).toBeDefined();
+          expect(cascadeQuery!.esql).toBe(
+            'FROM kibana_sample_data_logs | INLINE STATS count = COUNT(bytes), average = AVG(memory) BY clientip | WHERE clientip IS NULL'
+          );
+        });
       });
 
       describe('function group operations', () => {
@@ -968,6 +993,36 @@ describe('cascaded documents helpers utils', () => {
             'FROM kibana_sample_data_logs | STATS count = COUNT(bytes), average = AVG(memory) BY Named = CATEGORIZE(??field) | WHERE Named == "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)" | SORT average ASC'
           );
         });
+
+        it('uses postfix unary expression when "is_null" filter operation is applied regardless of the selected columns value', () => {
+          expect(
+            appendFilteringWhereClauseForCascadeLayout(
+              'FROM kibana_sample_data_logs | STATS count = COUNT(bytes), average = AVG(memory) BY clientip | SORT average ASC',
+              [],
+              dataViewMock,
+              'clientip',
+              GROUP_NOT_SET_VALUE,
+              'is_null'
+            )
+          ).toBe(
+            'FROM kibana_sample_data_logs | WHERE clientip IS NULL | STATS count = COUNT(bytes), average = AVG(memory) BY clientip | SORT average ASC'
+          );
+        });
+
+        it('uses postfix unary expression when "is_not_null" filter operation is applied regardless of the selected columns value', () => {
+          expect(
+            appendFilteringWhereClauseForCascadeLayout(
+              'FROM kibana_sample_data_logs | STATS count = COUNT(bytes), average = AVG(memory) BY clientip | SORT average ASC',
+              [],
+              dataViewMock,
+              'clientip',
+              GROUP_NOT_SET_VALUE,
+              'is_not_null'
+            )
+          ).toBe(
+            'FROM kibana_sample_data_logs | WHERE clientip IS NOT NULL | STATS count = COUNT(bytes), average = AVG(memory) BY clientip | SORT average ASC'
+          );
+        });
       });
     });
 

src/platform/packages/shared/kbn-esql-utils/src/utils/cascaded_documents_helpers/index.ts

@@ -54,6 +54,7 @@ import {
   requiresMatchPhrase,
   isCategorizeFunctionWithFunctionArgument,
 } from './utils';
+import { GROUP_NOT_SET_VALUE } from '../../../constants';
 
 const hasUnsupportedGroupingFunction = (definition: ESQLProperNode): boolean => {
   const funcExpr = isFunctionExpression(definition)
@@ -470,7 +471,11 @@ function handleStatsByColumnLeafOperation(
   const filterCommand = Builder.command({
     name: 'where',
     args: [
-      shouldUseMatchPhrase
+      operationValue === GROUP_NOT_SET_VALUE
+        ? Builder.expression.func.postfix('IS NULL', [
+            Builder.identifier({ name: operationColumnName }),
+          ])
+        : shouldUseMatchPhrase
         ? Builder.expression.func.call('match_phrase', [
             Builder.identifier({ name: operationColumnName }),
             Builder.expression.literal.string(operationValue as string),
@@ -848,6 +853,14 @@ export const appendFilteringWhereClauseForCascadeLayout = <
   if (isBinaryExpression(filteringExpression) && filteringExpression.name === 'and') {
     // This is already a combination of some conditions, for now we'll just append the new condition to the existing one
     modifiedFilteringWhereCommand = synth.cmd`WHERE ${computedFilteringExpression} AND ${filteringExpression}`;
+  } else if (
+    isFunctionExpression(filteringExpression) &&
+    filteringExpression.subtype === 'postfix-unary-expression'
+  ) {
+    modifiedFilteringWhereCommand =
+      (filteringExpression.args[0] as ESQLColumn).name === normalizedFieldName
+        ? synth.cmd`WHERE ${computedFilteringExpression}`
+        : synth.cmd`WHERE ${computedFilteringExpression} AND ${filteringExpression}`;
   } else {
     modifiedFilteringWhereCommand =
       isBinaryExpression(filteringExpression) &&

src/platform/plugins/shared/discover/public/application/main/components/layout/cascaded_documents/blocks/use_row_header_components.tsx

@@ -14,7 +14,11 @@ import type {
 } from '@elastic/eui';
 import type { EuiContextMenuPanelItemDescriptorEntry } from '@elastic/eui/src/components/context_menu/context_menu';
 import { type AggregateQuery } from '@kbn/es-query';
-import { appendFilteringWhereClauseForCascadeLayout, constructCascadeQuery } from '@kbn/esql-utils';
+import {
+  appendFilteringWhereClauseForCascadeLayout,
+  constructCascadeQuery,
+  GROUP_NOT_SET_VALUE,
+} from '@kbn/esql-utils';
 import { css } from '@emotion/react';
 import {
   EuiBadge,
@@ -100,7 +104,7 @@ const contextRowActions: Array<
         this.dataView,
         this.rowContext.groupId,
         this.rowContext.groupValue,
-        '+'
+        this.rowContext.groupValue === GROUP_NOT_SET_VALUE ? 'is_null' : '+'
       );
 
       if (!updatedQuery) {
@@ -126,7 +130,7 @@ const contextRowActions: Array<
         this.dataView,
         this.rowContext.groupId,
         this.rowContext.groupValue,
-        '-'
+        this.rowContext.groupValue === GROUP_NOT_SET_VALUE ? 'is_not_null' : '-'
       );
 
       if (!updatedQuery) {

src/platform/plugins/shared/discover/public/application/main/components/layout/cascaded_documents/hooks/data_fetching.test.ts

@@ -9,6 +9,7 @@
 
 import { renderHook, act } from '@testing-library/react';
 import type { DataTableRecord } from '@kbn/discover-utils';
+import { GROUP_NOT_SET_VALUE } from '@kbn/esql-utils';
 import { ESQLVariableType, type ESQLControlVariable } from '@kbn/esql-types';
 import type { AggregateQuery } from '@kbn/es-query';
 import { dataViewWithTimefieldMock } from '../../../../../../__mocks__/data_view_with_timefield';
@@ -99,7 +100,7 @@ describe('data_fetching related hooks', () => {
       expect(result.current.columnTypes.get('count')).toBe('number');
     });
 
-    it('should skip undefined and null values in grouping', () => {
+    it('should assign undefined and null values in grouping to the ES|QL unset value', () => {
       const mockRows = createMockRows([
         { category: 'A', count: 10 },
         { category: undefined, count: 5 },
@@ -116,9 +117,10 @@ describe('data_fetching related hooks', () => {
         })
       );
 
-      expect(result.current.data).toHaveLength(2);
+      expect(result.current.data).toHaveLength(3);
       expect(result.current.data.find((r) => r.groupValue === 'A')).toBeDefined();
       expect(result.current.data.find((r) => r.groupValue === 'B')).toBeDefined();
+      expect(result.current.data.find((r) => r.groupValue === GROUP_NOT_SET_VALUE)).toBeDefined();
     });
 
     it('should aggregate multiple applied functions', () => {

src/platform/plugins/shared/discover/public/application/main/components/layout/cascaded_documents/hooks/data_fetching.ts

@@ -8,7 +8,7 @@
  */
 
 import type { UnifiedDataTableProps } from '@kbn/unified-data-table';
-import { type ESQLStatsQueryMeta } from '@kbn/esql-utils/src/utils/cascaded_documents_helpers';
+import { GROUP_NOT_SET_VALUE, type ESQLStatsQueryMeta } from '@kbn/esql-utils';
 import { useMemo, useState } from 'react';
 import {
   type DataCascadeRowProps,
@@ -57,15 +57,10 @@ export const useGroupedCascadeData = ({
         }
 
         const rowsGroupedByValue = Object.groupBy(rows ?? [], (row) =>
-          String(row.flattened[resolvedGroupColumn])
+          String(row.flattened[resolvedGroupColumn] ?? GROUP_NOT_SET_VALUE)
         );
 
         Object.entries(rowsGroupedByValue).forEach(([groupValue, groupRows = []]) => {
-          // skip undefined and null values
-          if (groupValue === 'undefined' || groupValue === 'null') {
-            return;
-          }
-
           const groupNode: ESQLDataGroupNode = {
             id: uuidv5(`${groupColumn}-${groupValue}`, NODE_ID_NAMESPACE),
             // While we use explicit properties for better typing, the document_data_cascade package