[ML] Fixing results for unknown fields (#24577) (#24633)

jgowdyelastic · web-flow · commit 1b42f30d22e7 · 2018-10-31T12:17:12.000Z
* [Ml] Fixing results for known fields

* renaming known-type to unknown

* correcting comments

* tiny refactor
diff --git a/x-pack/plugins/ml/common/constants/field_types.js b/x-pack/plugins/ml/common/constants/field_types.js
@@ -27,7 +27,7 @@ export const ES_FIELD_TYPES = {
   TOKEN_COUNT: 'token_count',
   _ID: '_id',
   _SOURCE: '_source',
-  _TYPE: '_type'
+  _TYPE: '_type',
 };
 
 export const KBN_FIELD_TYPES = {
@@ -52,6 +52,7 @@ export const ML_JOB_FIELD_TYPES = {
   IP: 'ip',
   KEYWORD: 'keyword',
   NUMBER: 'number',
-  TEXT: 'text'
+  TEXT: 'text',
+  UNKNOWN: 'unknown',
 };
 
diff --git a/x-pack/plugins/ml/public/components/field_data_card/styles/main.less b/x-pack/plugins/ml/public/components/field_data_card/styles/main.less
@@ -34,7 +34,7 @@
     background-color: #920000;
   }
 
-  .type-other {
+  .type-other, .unknown {
     background-color: #bfa180;
   }
 
diff --git a/x-pack/plugins/ml/public/components/field_type_icon/field_type_icon.js b/x-pack/plugins/ml/public/components/field_type_icon/field_type_icon.js
@@ -47,6 +47,10 @@ export function FieldTypeIcon({ tooltipEnabled = false, type }) {
       ariaLabel = 'IP type';
       iconClass = 'fa-laptop';
       break;
+    case ML_JOB_FIELD_TYPES.UNKNOWN:
+      ariaLabel = 'Unknown type';
+      iconChar = '?';
+      break;
     default:
       // if type doesn't match one of ML_JOB_FIELD_TYPES
       // don't render the component at all
diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/_index.scss b/x-pack/plugins/ml/public/file_datavisualizer/components/_index.scss
@@ -4,3 +4,4 @@
 @import 'fields_stats/index';
 @import 'about_panel/index';
 @import 'import_summary/index';
+@import 'experimental_badge/index';
diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/about_panel/about_panel.js b/x-pack/plugins/ml/public/file_datavisualizer/components/about_panel/about_panel.js
@@ -52,7 +52,7 @@ export function AboutPanel({ onFilePickerChange }) {
 
 export function LoadingPanel() {
   return (
-    <EuiPage restrictWidth={200}>
+    <EuiPage restrictWidth={400}>
       <EuiPageBody>
         <EuiFlexGroup gutterSize="xl">
           <EuiFlexItem grow={false}>
diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/about_panel/welcome_content.js b/x-pack/plugins/ml/public/file_datavisualizer/components/about_panel/welcome_content.js
@@ -8,7 +8,6 @@
 import React from 'react';
 
 import {
-  EuiBetaBadge,
   EuiFlexGroup,
   EuiFlexItem,
   EuiIcon,
@@ -18,6 +17,8 @@ import {
   EuiTitle,
 } from '@elastic/eui';
 
+import { ExperimentalBadge } from '../experimental_badge';
+
 export function WelcomeContent() {
 
   return (
@@ -29,7 +30,9 @@ export function WelcomeContent() {
         <EuiTitle size="m">
           <h3>
             Visualize data from a log file &nbsp;
-            <EuiBetaBadge label="Experimental" tooltipContent="Experimental feature. We'd love to hear your feedback." />
+            <ExperimentalBadge
+              tooltipContent="Experimental feature. We'd love to hear your feedback."
+            />
           </h3>
         </EuiTitle>
         <EuiSpacer size="s" />
diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/experimental_badge/_experimental_badge.scss b/x-pack/plugins/ml/public/file_datavisualizer/components/experimental_badge/_experimental_badge.scss
@@ -0,0 +1,7 @@
+.ml-experimental-badge.euiBetaBadge {
+  font-size: 10px;
+  vertical-align: middle;
+  margin-bottom: 5px;
+  padding: 0px 20px;
+  line-height: 20px;
+}
diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/experimental_badge/_index.scss b/x-pack/plugins/ml/public/file_datavisualizer/components/experimental_badge/_index.scss
@@ -0,0 +1 @@
+@import 'experimental_badge'
diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/experimental_badge/experimental_badge.js b/x-pack/plugins/ml/public/file_datavisualizer/components/experimental_badge/experimental_badge.js
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+
+import {
+  EuiBetaBadge,
+} from '@elastic/eui';
+
+export function ExperimentalBadge({ tooltipContent }) {
+  return (
+    <span>
+      <EuiBetaBadge className="ml-experimental-badge" label="Experimental" tooltipContent={tooltipContent} />
+    </span>
+  );
+}
diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/experimental_badge/index.js b/x-pack/plugins/ml/public/file_datavisualizer/components/experimental_badge/index.js
@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+
+export { ExperimentalBadge } from './experimental_badge';
diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/fields_stats/field_stats_card.js b/x-pack/plugins/ml/public/file_datavisualizer/components/fields_stats/field_stats_card.js
@@ -9,7 +9,6 @@ import React from 'react';
 
 import {
   EuiSpacer,
-
 } from '@elastic/eui';
 
 import { FieldTypeIcon } from '../../../components/field_type_icon';
@@ -35,59 +34,69 @@ export function FieldStatsCard({ field }) {
           </div>
 
           <div className="card-contents">
-            <div className="stats">
-              <div className="stat">
-                <i className="fa fa-files-o" aria-hidden="true" /> {field.count} document{(field.count > 1) ? 's' : ''} ({percent}%)
-              </div>
-              <div className="stat">
-                <i className="fa fa-cubes" aria-hidden="true" /> {field.cardinality} distinct value{(field.cardinality > 1) ? 's' : ''}
-              </div>
-
-              {
-                (field.mean_value) &&
-                <React.Fragment>
-                  <div>
-                    <div className="stat min heading">min</div>
-                    <div className="stat median heading">median</div>
-                    <div className="stat max heading">max</div>
+            {(field.count > 0) &&
+              <React.Fragment>
+                <div className="stats">
+                  <div className="stat">
+                    <i className="fa fa-files-o" aria-hidden="true" /> {field.count} document{(field.count > 1) ? 's' : ''} ({percent}%)
                   </div>
-                  <div>
-                    <div className="stat min heading">{field.min_value}</div>
-                    <div className="stat median heading">{field.median_value}</div>
-                    <div className="stat max heading">{field.max_value}</div>
+                  <div className="stat">
+                    <i className="fa fa-cubes" aria-hidden="true" /> {field.cardinality} distinct value{(field.cardinality > 1) ? 's' : ''}
                   </div>
-                </React.Fragment>
-              }
-            </div>
-
-            {
-              (field.top_hits) &&
-              <React.Fragment>
-
-                <EuiSpacer size="s" />
 
-                <div className="stats">
-                  <div className="stat">top values</div>
-                  {field.top_hits.map(({ count, value }) => {
-                    const pcnt = Math.round(((count / field.count) * 100) * 100) / 100;
-                    return (
-                      <div key={value} className="top-value">
-                        <div className="field-label">{value}&nbsp;</div>
-                        <div className="top-value-bar-holder">
-                          <div
-                            className={`top-value-bar ${type}`}
-                            style={{ width: `${pcnt}%` }}
-                          />
-                        </div>
-                        <div className="count-label">{pcnt}%</div>
+                  {
+                    (field.mean_value) &&
+                    <React.Fragment>
+                      <div>
+                        <div className="stat min heading">min</div>
+                        <div className="stat median heading">median</div>
+                        <div className="stat max heading">max</div>
+                      </div>
+                      <div>
+                        <div className="stat min heading">{field.min_value}</div>
+                        <div className="stat median heading">{field.median_value}</div>
+                        <div className="stat max heading">{field.max_value}</div>
                       </div>
-                    );
+                    </React.Fragment>
                   }
-                  )}
                 </div>
+
+                {
+                  (field.top_hits) &&
+                  <React.Fragment>
+
+                    <EuiSpacer size="s" />
+
+                    <div className="stats">
+                      <div className="stat">top values</div>
+                      {field.top_hits.map(({ count, value }) => {
+                        const pcnt = Math.round(((count / field.count) * 100) * 100) / 100;
+                        return (
+                          <div key={value} className="top-value">
+                            <div className="field-label">{value}&nbsp;</div>
+                            <div className="top-value-bar-holder">
+                              <div
+                                className={`top-value-bar ${type}`}
+                                style={{ width: `${pcnt}%` }}
+                              />
+                            </div>
+                            <div className="count-label">{pcnt}%</div>
+                          </div>
+                        );
+                      }
+                      )}
+                    </div>
+                  </React.Fragment>
+                }
               </React.Fragment>
             }
-
+            {(field.count === 0) &&
+              <div className="stats">
+                <div className="stat">
+                  No field information available
+                </div>
+              </div>
+            }
           </div>
         </div>
       </div>
diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/fields_stats/fields_stats.js b/x-pack/plugins/ml/public/file_datavisualizer/components/fields_stats/fields_stats.js
@@ -10,6 +10,8 @@ import React, {
 } from 'react';
 
 import { FieldStatsCard } from './field_stats_card';
+import { getFieldNames } from './get_field_names';
+import { ML_JOB_FIELD_TYPES } from '../../../../common/constants/field_types';
 
 export class FieldsStats extends Component {
   constructor(props) {
@@ -46,26 +48,24 @@ function createFields(results) {
   const {
     mappings,
     field_stats: fieldStats,
-    column_names: columnNames,
     num_messages_analyzed: numMessagesAnalyzed,
     timestamp_field: timestampField,
   } = results;
 
   if (mappings && fieldStats) {
-    // if columnNames exists (i.e delimited) use it for the field list
-    // so we get the same order
-    const tempFields = (columnNames !== undefined) ? columnNames : Object.keys(fieldStats);
 
-    return tempFields.map((fName) => {
-      if (fieldStats[fName] !== undefined) {
-        const field = { name: fName };
-        const f  = fieldStats[fName];
-        const m  = mappings[fName];
+    const fieldNames = getFieldNames(results);
+
+    return fieldNames.map((name) => {
+      if (fieldStats[name] !== undefined) {
+        const field = { name };
+        const f  = fieldStats[name];
+        const m  = mappings[name];
 
         // sometimes the timestamp field is not in the mappings, and so our
         // collection of fields will be missing a time field with a type of date
-        if (fName === timestampField && field.type === undefined) {
-          field.type = 'date';
+        if (name === timestampField && field.type === undefined) {
+          field.type = ML_JOB_FIELD_TYPES.DATE;
         }
 
         if (f !== undefined) {
@@ -83,8 +83,15 @@ function createFields(results) {
 
         return field;
       } else {
+        // field is not in the field stats
+        // this could be the message field for a semi-structured log file or a
+        // field which the endpoint has not been able to work out any information for
+        const type = (mappings[name] && mappings[name].type === ML_JOB_FIELD_TYPES.TEXT) ?
+          ML_JOB_FIELD_TYPES.TEXT : ML_JOB_FIELD_TYPES.UNKNOWN;
+
         return {
-          name: fName,
+          name,
+          type,
           mean_value: 0,
           count: 0,
           cardinality: 0,
diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/fields_stats/get_field_names.js b/x-pack/plugins/ml/public/file_datavisualizer/components/fields_stats/get_field_names.js
@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+
+import { difference } from 'lodash';
+
+export function getFieldNames(results) {
+  const {
+    mappings,
+    field_stats: fieldStats,
+    column_names: columnNames,
+  } = results;
+
+  // if columnNames exists (i.e delimited) use it for the field list
+  // so we get the same order
+  const tempFields = (columnNames !== undefined) ? columnNames : Object.keys(fieldStats);
+
+
+  // there may be fields in the mappings which do not exist in the field_stats
+  // e.g. the message field for a semi-structured log file, as they have no stats.
+  // add any extra fields to the list
+  const differenceFields = difference(Object.keys(mappings), tempFields);
+
+  // except @timestamp
+  const timestampIndex = differenceFields.indexOf('@timestamp');
+  if (timestampIndex !== -1) {
+    differenceFields.splice(timestampIndex, 1);
+  }
+
+  if (differenceFields.length) {
+    tempFields.push(...differenceFields);
+  }
+  return tempFields;
+}
diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/import_view/import_view.js b/x-pack/plugins/ml/public/file_datavisualizer/components/import_view/import_view.js
@@ -10,7 +10,6 @@ import React, {
 } from 'react';
 
 import {
-  EuiBetaBadge,
   EuiButton,
   EuiSpacer,
   EuiPanel,
@@ -23,6 +22,7 @@ import { ImportProgress, IMPORT_STATUS } from '../import_progress';
 import { ImportErrors } from '../import_errors';
 import { ImportSummary } from '../import_summary';
 import { ImportSettings } from '../import_settings';
+import { ExperimentalBadge } from '../experimental_badge';
 import { getIndexPatternNames, refreshIndexPatterns } from '../../../util/index_utils';
 import { ml } from '../../../services/ml_api_service';
 
@@ -325,7 +325,9 @@ export class ImportView extends Component {
           <EuiTitle size="s">
             <h3>
               Import data &nbsp;
-              <EuiBetaBadge label="Experimental" tooltipContent="Experimental feature. We'd love to hear your feedback." />
+              <ExperimentalBadge
+                tooltipContent="Experimental feature. We'd love to hear your feedback."
+              />
             </h3>
           </EuiTitle>
 

Original file line number	Diff line number	Diff line change
`@@ -34,7 +34,7 @@`
`34`	`34`	`background-color: #920000;`
`35`	`35`	`}`
`36`	`36`
`37`		`- .type-other {`
	`37`	`+ .type-other, .unknown {`
`38`	`38`	`background-color: #bfa180;`
`39`	`39`	`}`
`40`	`40`