[Lens] Log data tables properly

src/plugins/chart_expressions/expression_gauge/common/expression_functions/gauge_function.ts

@@ -182,12 +182,16 @@ export const gaugeFunction = (): GaugeExpressionFunctionDefinition => ({
     }
 
     if (handlers?.inspectorAdapters?.tables) {
-      const logTable = prepareLogTable(data, [
-        [metric ? [metric] : undefined, strings.getMetricHelp()],
-        [min ? [min] : undefined, strings.getMinHelp()],
-        [max ? [max] : undefined, strings.getMaxHelp()],
-        [goal ? [goal] : undefined, strings.getGoalHelp()],
-      ]);
+      const logTable = prepareLogTable(
+        data,
+        [
+          [metric ? [metric] : undefined, strings.getMetricHelp()],
+          [min ? [min] : undefined, strings.getMinHelp()],
+          [max ? [max] : undefined, strings.getMaxHelp()],
+          [goal ? [goal] : undefined, strings.getGoalHelp()],
+        ],
+        true
+      );
 
       handlers.inspectorAdapters.tables.logDatatable('default', logTable);
     }

src/plugins/chart_expressions/expression_heatmap/common/expression_functions/heatmap_function.ts

@@ -28,7 +28,7 @@ const convertToVisDimension = (
   const column = columns.find((c) => c.id === accessor);
   if (!column) return;
   return {
-    accessor: Number(column.id),
+    accessor: column,
     format: {
       id: column.meta.params?.id,
       params: { ...column.meta.params?.params },
@@ -212,7 +212,7 @@ export const heatmapFunction = (): HeatmapExpressionFunctionDefinition => ({
           })
         );
       }
-      const logTable = prepareLogTable(data, argsTable);
+      const logTable = prepareLogTable(data, argsTable, true);
       handlers.inspectorAdapters.tables.logDatatable('default', logTable);
     }
     return {

src/plugins/chart_expressions/expression_metric/common/expression_functions/metric_vis_function.ts

@@ -162,7 +162,7 @@ export const metricVisFunction = (): MetricVisExpressionFunctionDefinition => ({
           }),
         ]);
       }
-      const logTable = prepareLogTable(input, argsTable);
+      const logTable = prepareLogTable(input, argsTable, true);
       handlers.inspectorAdapters.tables.logDatatable('default', logTable);
     }
 

src/plugins/chart_expressions/expression_partition_vis/common/expression_functions/mosaic_vis_function.ts

@@ -134,12 +134,16 @@ export const mosaicVisFunction = (): MosaicVisExpressionFunctionDefinition => ({
     };
 
     if (handlers?.inspectorAdapters?.tables) {
-      const logTable = prepareLogTable(context, [
-        [[args.metric], strings.getSliceSizeHelp()],
-        [args.buckets, strings.getSliceHelp()],
-        [args.splitColumn, strings.getColumnSplitHelp()],
-        [args.splitRow, strings.getRowSplitHelp()],
-      ]);
+      const logTable = prepareLogTable(
+        context,
+        [
+          [[args.metric], strings.getSliceSizeHelp()],
+          [args.buckets, strings.getSliceHelp()],
+          [args.splitColumn, strings.getColumnSplitHelp()],
+          [args.splitRow, strings.getRowSplitHelp()],
+        ],
+        true
+      );
       handlers.inspectorAdapters.tables.logDatatable('default', logTable);
     }
 

src/plugins/chart_expressions/expression_partition_vis/common/expression_functions/pie_vis_function.ts

@@ -154,12 +154,16 @@ export const pieVisFunction = (): PieVisExpressionFunctionDefinition => ({
     };
 
     if (handlers?.inspectorAdapters?.tables) {
-      const logTable = prepareLogTable(context, [
-        [[args.metric], strings.getSliceSizeHelp()],
-        [args.buckets, strings.getSliceHelp()],
-        [args.splitColumn, strings.getColumnSplitHelp()],
-        [args.splitRow, strings.getRowSplitHelp()],
-      ]);
+      const logTable = prepareLogTable(
+        context,
+        [
+          [[args.metric], strings.getSliceSizeHelp()],
+          [args.buckets, strings.getSliceHelp()],
+          [args.splitColumn, strings.getColumnSplitHelp()],
+          [args.splitRow, strings.getRowSplitHelp()],
+        ],
+        true
+      );
       handlers.inspectorAdapters.tables.logDatatable('default', logTable);
     }
 

src/plugins/chart_expressions/expression_partition_vis/common/expression_functions/treemap_vis_function.ts

@@ -134,12 +134,16 @@ export const treemapVisFunction = (): TreemapVisExpressionFunctionDefinition =>
     };
 
     if (handlers?.inspectorAdapters?.tables) {
-      const logTable = prepareLogTable(context, [
-        [[args.metric], strings.getSliceSizeHelp()],
-        [args.buckets, strings.getSliceHelp()],
-        [args.splitColumn, strings.getColumnSplitHelp()],
-        [args.splitRow, strings.getRowSplitHelp()],
-      ]);
+      const logTable = prepareLogTable(
+        context,
+        [
+          [[args.metric], strings.getSliceSizeHelp()],
+          [args.buckets, strings.getSliceHelp()],
+          [args.splitColumn, strings.getColumnSplitHelp()],
+          [args.splitRow, strings.getRowSplitHelp()],
+        ],
+        true
+      );
       handlers.inspectorAdapters.tables.logDatatable('default', logTable);
     }
 

src/plugins/chart_expressions/expression_partition_vis/common/expression_functions/waffle_vis_function.ts

@@ -129,12 +129,16 @@ export const waffleVisFunction = (): WaffleVisExpressionFunctionDefinition => ({
     };
 
     if (handlers?.inspectorAdapters?.tables) {
-      const logTable = prepareLogTable(context, [
-        [[args.metric], strings.getSliceSizeHelp()],
-        [buckets, strings.getSliceHelp()],
-        [args.splitColumn, strings.getColumnSplitHelp()],
-        [args.splitRow, strings.getRowSplitHelp()],
-      ]);
+      const logTable = prepareLogTable(
+        context,
+        [
+          [[args.metric], strings.getSliceSizeHelp()],
+          [buckets, strings.getSliceHelp()],
+          [args.splitColumn, strings.getColumnSplitHelp()],
+          [args.splitRow, strings.getRowSplitHelp()],
+        ],
+        true
+      );
       handlers.inspectorAdapters.tables.logDatatable('default', logTable);
     }
 

src/plugins/chart_expressions/expression_tagcloud/common/expression_functions/tagcloud_function.ts

@@ -164,7 +164,7 @@ export const tagcloudFunction: ExpressionTagcloudFunction = () => {
         if (args.bucket) {
           argsTable.push([[args.bucket], dimension.tags]);
         }
-        const logTable = prepareLogTable(input, argsTable);
+        const logTable = prepareLogTable(input, argsTable, true);
         handlers.inspectorAdapters.tables.logDatatable('default', logTable);
       }
       return {

src/plugins/visualizations/common/utils/prepare_log_table.ts

@@ -57,17 +57,23 @@ const getDimensionName = (
   }
 };
 
-export const prepareLogTable = (datatable: Datatable, dimensions: Dimension[]) => {
+export const prepareLogTable = (
+  datatable: Datatable,
+  dimensions: Dimension[],
+  removeUnmappedColumns: boolean = false
+) => {
   return {
     ...datatable,
-    columns: datatable.columns.map((column, columnIndex) => {
-      return {
-        ...column,
-        meta: {
-          ...column.meta,
-          dimensionName: getDimensionName(column, columnIndex, dimensions),
-        },
-      };
-    }),
+    columns: datatable.columns
+      .map((column, columnIndex) => {
+        return {
+          ...column,
+          meta: {
+            ...column.meta,
+            dimensionName: getDimensionName(column, columnIndex, dimensions),
+          },
+        };
+      })
+      .filter((column) => !removeUnmappedColumns || column.meta.dimensionName),
   };
 };

x-pack/plugins/lens/common/expressions/datatable/datatable_fn.ts

@@ -6,6 +6,8 @@
  */
 
 import { cloneDeep } from 'lodash';
+import { i18n } from '@kbn/i18n';
+import { prepareLogTable } from '../../../../../../src/plugins/visualizations/common/utils';
 import { FormatFactory, LensMultiTable } from '../../types';
 import { transposeTable } from './transpose_helpers';
 import { computeSummaryRowForColumn } from './summary';
@@ -15,7 +17,6 @@ import type {
   ExecutionContext,
 } from '../../../../../../src/plugins/expressions';
 import type { DatatableExpressionFunction } from './types';
-import { logDataTable } from '../expressions_utils';
 
 function isRange(meta: { params?: { id?: string } } | undefined) {
   return meta?.params?.id === 'range';
@@ -26,13 +27,26 @@ export const datatableFn =
     getFormatFactory: (context: ExecutionContext) => FormatFactory | Promise<FormatFactory>
   ): DatatableExpressionFunction['fn'] =>
   async (data, args, context) => {
+    const [firstTable] = Object.values(data.tables);
     if (context?.inspectorAdapters?.tables) {
-      logDataTable(context.inspectorAdapters.tables, data.tables);
+      const logTable = prepareLogTable(
+        Object.values(data.tables)[0],
+        [
+          [
+            args.columns.map((column) => column.columnId),
+            i18n.translate('xpack.lens.datatable.column.help', {
+              defaultMessage: 'Datatable column',
+            }),
+          ],
+        ],
+        true
+      );
+
+      context.inspectorAdapters.tables.logDatatable('default', logTable);
     }
 
     let untransposedData: LensMultiTable | undefined;
     // do the sorting at this level to propagate it also at CSV download
-    const [firstTable] = Object.values(data.tables);
     const [layerId] = Object.keys(context.inspectorAdapters.tables || {});
     const formatters: Record<string, ReturnType<FormatFactory>> = {};
     const formatFactory = await getFormatFactory(context);

x-pack/plugins/lens/common/expressions/xy_chart/xy_chart.ts

@@ -10,14 +10,33 @@ import type { ExpressionValueSearchContext } from '../../../../../../src/plugins
 import type { LensMultiTable } from '../../types';
 import type { XYArgs } from './xy_args';
 import { fittingFunctionDefinitions } from './fitting_function';
+import { prepareLogTable } from '../../../../../../src/plugins/visualizations/common/utils';
 import { endValueDefinitions } from './end_value';
-import { logDataTable } from '../expressions_utils';
 
 export interface XYChartProps {
   data: LensMultiTable;
   args: XYArgs;
 }
 
+const strings = {
+  getMetricHelp: () =>
+    i18n.translate('xpack.lens.xy.logDatatable.metric', {
+      defaultMessage: 'Vertical axis',
+    }),
+  getXAxisHelp: () =>
+    i18n.translate('xpack.lens.xy.logDatatable.x', {
+      defaultMessage: 'Horizontal axis',
+    }),
+  getBreakdownHelp: () =>
+    i18n.translate('xpack.lens.xy.logDatatable.breakDown', {
+      defaultMessage: 'Break down by',
+    }),
+  getReferenceLineHelp: () =>
+    i18n.translate('xpack.lens.xy.logDatatable.breakDown', {
+      defaultMessage: 'Break down by',
+    }),
+};
+
 export interface XYRender {
   type: 'render';
   as: 'lens_xy_chart_renderer';
@@ -174,7 +193,26 @@ export const xyChart: ExpressionFunctionDefinition<
   },
   fn(data: LensMultiTable, args: XYArgs, handlers) {
     if (handlers?.inspectorAdapters?.tables) {
-      logDataTable(handlers.inspectorAdapters.tables, data.tables);
+      args.layers.forEach((layer) => {
+        if (layer.layerType === 'annotations') {
+          return;
+        }
+        const { layerId, accessors, xAccessor, splitAccessor, layerType } = layer;
+        const logTable = prepareLogTable(
+          data.tables[layerId],
+          [
+            [
+              accessors ? accessors : undefined,
+              layerType === 'data' ? strings.getMetricHelp() : strings.getReferenceLineHelp(),
+            ],
+            [xAccessor ? [xAccessor] : undefined, strings.getXAxisHelp()],
+            [splitAccessor ? [splitAccessor] : undefined, strings.getBreakdownHelp()],
+          ],
+          true
+        );
+
+        handlers.inspectorAdapters.tables.logDatatable(layerId, logTable);
+      });
     }
     return {
       type: 'render',