[8.19] [ColorMapping] Store assignments as raw/serialized values (#207957)

packages/kbn-optimizer/limits.yml

@@ -9,7 +9,7 @@ pageLoadAssetSize:
   bfetch: 22837
   canvas: 29355
   cases: 180037
-  charts: 55000
+  charts: 60000
   cloud: 21076
   cloudDataMigration: 19170
   cloudDefend: 18697

src/platform/packages/shared/chart-expressions-common/color_categories.test.ts

@@ -7,55 +7,148 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { DatatableColumn, DatatableRow } from '@kbn/expressions-plugin/common';
-import { getColorCategories } from './color_categories';
-
-const getNextExtension = (() => {
-  let i = 0;
-  const extensions = ['gz', 'css', '', 'rpm', 'deb', 'zip', null];
-  return () => extensions[i++ % extensions.length];
-})();
-
-const basicDatatable = {
-  columns: ['count', 'extension'].map((id) => ({ id } as DatatableColumn)),
-  rows: Array.from({ length: 10 }).map((_, i) => ({
-    count: i,
-    extension: getNextExtension(),
-  })) as DatatableRow[],
-};
-
-describe('getColorCategories', () => {
-  it('should return no categories when accessor is undefined', () => {
-    expect(getColorCategories(basicDatatable.rows)).toEqual([]);
-  });
+import { DatatableRow } from '@kbn/expressions-plugin/common';
+import { getColorCategories, getLegacyColorCategories } from './color_categories';
+import { MultiFieldKey, RangeKey } from '@kbn/data-plugin/common';
 
-  it('should return no categories when accessor is not found', () => {
-    expect(getColorCategories(basicDatatable.rows, 'N/A')).toEqual([]);
-  });
+class FakeClass {}
 
-  it('should return no categories when no rows are defined', () => {
-    expect(getColorCategories(undefined, 'extension')).toEqual([]);
-  });
+const values = [
+  1,
+  false,
+  true,
+  0,
+  NaN,
+  null,
+  undefined,
+  '',
+  'test-string',
+  { test: 'obj' },
+  ['array'],
+];
+const mockRange = new RangeKey({ from: 0, to: 100 });
+const mockMultiField = new MultiFieldKey({ key: ['one', 'two'] });
+const fakeClass = new FakeClass();
+const complex = [mockRange, mockMultiField, fakeClass];
+
+const mockDatatableRows = Array.from({ length: 20 }).map<DatatableRow>((_, i) => ({
+  count: i,
+  value: values[i % values.length],
+  complex: complex[i % complex.length],
+}));
+
+describe('Color Categories', () => {
+  describe('getColorCategories', () => {
+    it('should return no categories when accessor is undefined', () => {
+      expect(getColorCategories(mockDatatableRows)).toEqual([]);
+    });
+
+    it('should return no categories when accessor is not found', () => {
+      expect(getColorCategories(mockDatatableRows, 'N/A')).toEqual([]);
+    });
+
+    it('should return no categories when no rows are defined', () => {
+      expect(getColorCategories(undefined, 'extension')).toEqual([]);
+    });
 
-  it('should return all categories from non-transpose datatable', () => {
-    expect(getColorCategories(basicDatatable.rows, 'extension')).toEqual([
-      'gz',
-      'css',
-      '',
-      'rpm',
-      'deb',
-      'zip',
-      'null',
-    ]);
+    it('should return all categories from mixed value datatable', () => {
+      expect(getColorCategories(mockDatatableRows, 'value')).toEqual([
+        1,
+        false,
+        true,
+        0,
+        NaN,
+        null,
+        undefined,
+        '',
+        'test-string',
+        {
+          test: 'obj',
+        },
+        ['array'],
+      ]);
+    });
+
+    it('should exclude selected categories from datatable', () => {
+      expect(
+        getColorCategories(mockDatatableRows, 'value', [
+          1,
+          false,
+          true,
+          0,
+          NaN,
+          null,
+          undefined,
+          '',
+        ])
+      ).toEqual([
+        'test-string',
+        {
+          test: 'obj',
+        },
+        ['array'],
+      ]);
+    });
+
+    it('should return known serialized categories from datatable', () => {
+      expect(getColorCategories(mockDatatableRows, 'complex', [])).toEqual([
+        mockRange.serialize(),
+        mockMultiField.serialize(),
+        fakeClass,
+      ]);
+    });
   });
 
-  it('should exclude selected categories from non-transpose datatable', () => {
-    expect(getColorCategories(basicDatatable.rows, 'extension', ['', null])).toEqual([
-      'gz',
-      'css',
-      'rpm',
-      'deb',
-      'zip',
-    ]);
+  describe('getLegacyColorCategories', () => {
+    it('should return no categories when accessor is undefined', () => {
+      expect(getLegacyColorCategories(mockDatatableRows)).toEqual([]);
+    });
+
+    it('should return no categories when accessor is not found', () => {
+      expect(getLegacyColorCategories(mockDatatableRows, 'N/A')).toEqual([]);
+    });
+
+    it('should return no categories when no rows are defined', () => {
+      expect(getLegacyColorCategories(undefined, 'extension')).toEqual([]);
+    });
+
+    it('should return all categories from mixed value datatable', () => {
+      expect(getLegacyColorCategories(mockDatatableRows, 'value')).toEqual([
+        '1',
+        'false',
+        'true',
+        '0',
+        'NaN',
+        'null',
+        'undefined',
+        '',
+        'test-string',
+        '{"test":"obj"}',
+        'array',
+      ]);
+    });
+
+    it('should exclude selected categories from datatable', () => {
+      expect(
+        getLegacyColorCategories(mockDatatableRows, 'value', [
+          1,
+          false,
+          true,
+          0,
+          NaN,
+          null,
+          undefined,
+          '',
+        ])
+      ).toEqual(['test-string', '{"test":"obj"}', 'array']);
+    });
+
+    it('should return known serialized categories from datatable', () => {
+      expect(getLegacyColorCategories(mockDatatableRows, 'complex', [])).toEqual([
+        String(mockRange),
+        String(mockMultiField),
+        JSON.stringify(fakeClass),
+      ]);
+    });
   });
 });

src/platform/packages/shared/chart-expressions-common/color_categories.ts

@@ -8,40 +8,46 @@
  */
 
 import { DatatableRow } from '@kbn/expressions-plugin/common';
-import { isMultiFieldKey } from '@kbn/data-plugin/common';
+import { RawValue, SerializedValue, serializeField } from '@kbn/data-plugin/common';
+import { getValueKey } from '@kbn/coloring';
 
 /**
- * Get the stringified version of all the categories that needs to be colored in the chart.
- * Multifield keys will return as array of string and simple fields (numeric, string) will be returned as a plain unformatted string.
+ * Returns all serialized categories of the dataset for color matching.
+ * All non-serializable fields will be as a plain unformatted string.
  *
  * Note: This does **NOT** support transposed columns
  */
 export function getColorCategories(
   rows: DatatableRow[] = [],
   accessor?: string,
-  exclude?: any[]
-): Array<string | string[]> {
+  exclude?: RawValue[],
+  legacyMode: boolean = false // stringifies raw values
+): SerializedValue[] {
   if (!accessor) return [];
 
-  return rows
-    .filter(({ [accessor]: v }) => !(v === undefined || exclude?.includes(v)))
-    .map((r) => {
-      const v = r[accessor];
-      // The categories needs to be stringified in their unformatted version.
-      // We can't distinguish between a number and a string from a text input and the match should
-      // work with both numeric field values and string values.
-      const key = (isMultiFieldKey(v) ? v.keys : [v]).map(String);
-      const stringifiedKeys = key.join(',');
-      return { key, stringifiedKeys };
-    })
-    .reduce<{ keys: Set<string>; categories: Array<string | string[]> }>(
-      (acc, { key, stringifiedKeys }) => {
-        if (!acc.keys.has(stringifiedKeys)) {
-          acc.keys.add(stringifiedKeys);
-          acc.categories.push(key.length === 1 ? key[0] : key);
-        }
-        return acc;
-      },
-      { keys: new Set(), categories: [] }
-    ).categories;
+  const seen = new Set<unknown>();
+  return rows.reduce<SerializedValue[]>((acc, row) => {
+    const hasValue = Object.hasOwn(row, accessor);
+    const rawValue: RawValue = row[accessor];
+    const key = getValueKey(rawValue);
+    if (hasValue && !exclude?.includes(rawValue) && !seen.has(key)) {
+      const value = serializeField(rawValue);
+      seen.add(key);
+      acc.push(legacyMode ? key : value);
+    }
+    return acc;
+  }, []);
+}
+
+/**
+ * Returns all *stringified* categories of the dataset for color matching.
+ *
+ * Should **only** be used with legacy `palettes`
+ */
+export function getLegacyColorCategories(
+  rows?: DatatableRow[],
+  accessor?: string,
+  exclude?: RawValue[]
+): string[] {
+  return getColorCategories(rows, accessor, exclude, true).map(String);
 }

src/platform/packages/shared/chart-expressions-common/index.ts

@@ -15,4 +15,4 @@ export {
 } from './utils';
 export type { Simplify, MakeOverridesSerializable, ChartSizeSpec, ChartSizeEvent } from './types';
 export { isChartSizeEvent } from './types';
-export { getColorCategories } from './color_categories';
+export { getColorCategories, getLegacyColorCategories } from './color_categories';

src/platform/packages/shared/chart-expressions-common/tsconfig.json

@@ -19,5 +19,6 @@
     "@kbn/core-execution-context-common",
     "@kbn/expressions-plugin",
     "@kbn/data-plugin",
+    "@kbn/coloring",
   ]
 }

src/platform/packages/shared/kbn-coloring/src/shared_components/color_mapping/README.md

@@ -3,9 +3,9 @@
 This shared component can be used to define a color mapping as an association of one or multiple string values to a color definition.
 
 This package provides:
-- a React component, called `CategoricalColorMapping` that provides a simplified UI (that in general can be hosted in a flyout), that helps the user generate a `ColorMapping.Config` object that descibes the mappings configuration
-- a function `getColorFactory` that given a color mapping configuration returns a function that maps a passed category to the corresponding color
-- a definition scheme for the color mapping, based on the type `ColorMapping.Config`, that provides an extensible way of describing the link between colors and rules. Collects the minimal information required apply colors based on categories. Together with the `ColorMappingInputData` can be used to get colors in a deterministic way.
+- A React component, called `CategoricalColorMapping` that provides a simplified UI (that in general can be hosted in a flyout), that helps the user generate a `ColorMapping.Config` object that describes the mappings configuration
+- A function `getColorFactory` that given a color mapping configuration returns a function that maps a passed category to the corresponding color
+- A definition scheme for the color mapping, based on the type `ColorMapping.Config`, that provides an extensible way of describing the link between colors and rules. Collects the minimal information required apply colors based on categories. Together with the `ColorMappingInputData` can be used to get colors in a deterministic way.
 
 
 An example of the configuration is the following:
@@ -14,10 +14,10 @@ const DEFAULT_COLOR_MAPPING_CONFIG: ColorMapping.Config = {
   assignmentMode: 'auto',
   assignments: [
     {
-        rule: {
-            type: 'matchExactly',
-            values: [''];
-        },
+        rules: [{
+            type: 'match',
+            pattern: '';
+        }],
         color: {
             type: 'categorical',
             paletteId: 'eui',
@@ -27,9 +27,9 @@ const DEFAULT_COLOR_MAPPING_CONFIG: ColorMapping.Config = {
   ],
   specialAssignments: [
     {
-      rule: {
+      rules: [{
         type: 'other',
-      },
+      }],
       color: {
         type: 'categorical',
         paletteId: 'neutral',
@@ -45,7 +45,7 @@ const DEFAULT_COLOR_MAPPING_CONFIG: ColorMapping.Config = {
 };
 ```
 
-The function `getColorFactory` is a curry function where, given the model, a palette getter, the theme mode (dark/light) and a list of categories, returns a function that can be used to pick the right color based on a given category.
+The function `getColorFactory` is a curry function where, given the model, a palette getter, the theme mode (dark/light) and a list of categories, returns a `ColorHandlingFn` that can be used to pick the right color based on a given category.
 
 ```ts
 function getColorFactory(
@@ -56,22 +56,19 @@ function getColorFactory(
       type: 'categories';
       categories: Array<string | string[]>;
     }
-): (category: string | string[]) => Color
+): ColorHandlingFn
 ```
 
-
-
 A `category` can be in the shape of a plain string or an array of strings. Numbers, MultiFieldKey, IP etc needs to be stringified.
 
-
 The `CategoricalColorMapping` React component has the following props:
 
 ```tsx
 function CategoricalColorMapping(props: {
   /** The initial color mapping model, usually coming from a the visualization saved object */
   model: ColorMapping.Config;
-  /** A map of paletteId and palette configuration */
-  palettes: Map<string, ColorMapping.CategoricalPalette>;
+  /** A collection of palette configurations */
+  palettes: KbnPalettes;
   /** A data description of what needs to be colored */
   data: ColorMappingInputData;
   /** Theme dark mode */
@@ -80,8 +77,11 @@ function CategoricalColorMapping(props: {
   specialTokens: Map<string, string>;
   /** A function called at every change in the model */
   onModelUpdate: (model: ColorMapping.Config) => void;
+  /** Formatter for raw value assignments */
+  formatter?: IFieldFormat;
+  /** Allow custom match rule when no other option is found */
+  allowCustomMatch?: boolean;
 })
-
 ```
 
-the `onModelUpdate` callback is called everytime a change in the model is applied from within the component. Is not called when the `model` prop is updated.
+the `onModelUpdate` callback is called every time a change in the model is applied from within the component. Is not called when the `model` prop is updated.