Add Managed label to data streams and a view switch for the table

x-pack/plugins/index_management/__jest__/client_integration/helpers/test_subjects.ts

@@ -21,6 +21,7 @@ export type TestSubjects =
   | 'filterList.filterItem'
   | 'ilmPolicyLink'
   | 'includeStatsSwitch'
+  | 'includeManagedSwitch'
   | 'indexTable'
   | 'indexTableIncludeHiddenIndicesToggle'
   | 'indexTableIndexNameLink'

x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.helpers.ts

@@ -19,6 +19,7 @@ export interface DataStreamsTabTestBed extends TestBed<TestSubjects> {
     goToDataStreamsList: () => void;
     clickEmptyPromptIndexTemplateLink: () => void;
     clickIncludeStatsSwitch: () => void;
+    clickIncludeManagedSwitch: () => void;
     clickReloadButton: () => void;
     clickNameAt: (index: number) => void;
     clickIndicesAt: (index: number) => void;
@@ -80,6 +81,11 @@ export const setup = async (overridingDependencies: any = {}): Promise<DataStrea
     find('includeStatsSwitch').simulate('click');
   };
 
+  const clickIncludeManagedSwitch = () => {
+    const { find } = testBed;
+    find('includeManagedSwitch').simulate('click');
+  };
+
   const clickReloadButton = () => {
     const { find } = testBed;
     find('reloadButton').simulate('click');
@@ -183,6 +189,7 @@ export const setup = async (overridingDependencies: any = {}): Promise<DataStrea
       goToDataStreamsList,
       clickEmptyPromptIndexTemplateLink,
       clickIncludeStatsSwitch,
+      clickIncludeManagedSwitch,
       clickReloadButton,
       clickNameAt,
       clickIndicesAt,
@@ -202,8 +209,8 @@ export const setup = async (overridingDependencies: any = {}): Promise<DataStrea
   };
 };
 
-export const createDataStreamPayload = (name: string): DataStream => ({
-  name,
+export const createDataStreamPayload = (dataStream: Partial<DataStream>): DataStream => ({
+  name: 'my-data-stream',
   timeStampField: { name: '@timestamp' },
   indices: [
     {
@@ -216,6 +223,7 @@ export const createDataStreamPayload = (name: string): DataStream => ({
   indexTemplateName: 'indexTemplate',
   storageSize: '1b',
   maxTimeStamp: 420,
+  ...dataStream,
 });
 
 export const createDataStreamBackingIndex = (indexName: string, dataStreamName: string) => ({

x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.test.ts

@@ -99,10 +99,10 @@ describe('Data Streams tab', () => {
         createNonDataStreamIndex('non-data-stream-index'),
       ]);
 
-      const dataStreamForDetailPanel = createDataStreamPayload('dataStream1');
+      const dataStreamForDetailPanel = createDataStreamPayload({ name: 'dataStream1' });
       setLoadDataStreamsResponse([
         dataStreamForDetailPanel,
-        createDataStreamPayload('dataStream2'),
+        createDataStreamPayload({ name: 'dataStream2' }),
       ]);
       setLoadDataStreamResponse(dataStreamForDetailPanel);
 
@@ -287,9 +287,9 @@ describe('Data Streams tab', () => {
         createDataStreamBackingIndex('data-stream-index2', 'dataStream2'),
       ]);
 
-      const dataStreamDollarSign = createDataStreamPayload('%dataStream');
-      setLoadDataStreamsResponse([dataStreamDollarSign]);
-      setLoadDataStreamResponse(dataStreamDollarSign);
+      const dataStreamPercentSign = createDataStreamPayload({ name: '%dataStream' });
+      setLoadDataStreamsResponse([dataStreamPercentSign]);
+      setLoadDataStreamResponse(dataStreamPercentSign);
 
       testBed = await setup({
         history: createMemoryHistory(),
@@ -327,10 +327,10 @@ describe('Data Streams tab', () => {
     test('with an ILM url generator and an ILM policy', async () => {
       const { setLoadDataStreamsResponse, setLoadDataStreamResponse } = httpRequestsMockHelpers;
 
-      const dataStreamForDetailPanel = {
-        ...createDataStreamPayload('dataStream1'),
+      const dataStreamForDetailPanel = createDataStreamPayload({
+        name: 'dataStream1',
         ilmPolicyName: 'my_ilm_policy',
-      };
+      });
       setLoadDataStreamsResponse([dataStreamForDetailPanel]);
       setLoadDataStreamResponse(dataStreamForDetailPanel);
 
@@ -351,7 +351,7 @@ describe('Data Streams tab', () => {
     test('with an ILM url generator and no ILM policy', async () => {
       const { setLoadDataStreamsResponse, setLoadDataStreamResponse } = httpRequestsMockHelpers;
 
-      const dataStreamForDetailPanel = createDataStreamPayload('dataStream1');
+      const dataStreamForDetailPanel = createDataStreamPayload({ name: 'dataStream1' });
       setLoadDataStreamsResponse([dataStreamForDetailPanel]);
       setLoadDataStreamResponse(dataStreamForDetailPanel);
 
@@ -373,10 +373,10 @@ describe('Data Streams tab', () => {
     test('without an ILM url generator and with an ILM policy', async () => {
       const { setLoadDataStreamsResponse, setLoadDataStreamResponse } = httpRequestsMockHelpers;
 
-      const dataStreamForDetailPanel = {
-        ...createDataStreamPayload('dataStream1'),
+      const dataStreamForDetailPanel = createDataStreamPayload({
+        name: 'dataStream1',
         ilmPolicyName: 'my_ilm_policy',
-      };
+      });
       setLoadDataStreamsResponse([dataStreamForDetailPanel]);
       setLoadDataStreamResponse(dataStreamForDetailPanel);
 
@@ -395,4 +395,58 @@ describe('Data Streams tab', () => {
       expect(findDetailPanelIlmPolicyName().contains('my_ilm_policy')).toBeTruthy();
     });
   });
+
+  describe('managed data streams', () => {
+    const nonBreakingSpace = ' ';
+    beforeEach(async () => {
+      const managedDataStream = createDataStreamPayload({
+        name: 'managed-data-stream',
+        _meta: {
+          package: 'test',
+          managed: true,
+          managed_by: 'ingest-manager',
+        },
+      });
+      const nonManagedDataStream = createDataStreamPayload({ name: 'non-managed-data-stream' });
+      httpRequestsMockHelpers.setLoadDataStreamsResponse([managedDataStream, nonManagedDataStream]);
+
+      testBed = await setup({
+        history: createMemoryHistory(),
+      });
+      await act(async () => {
+        testBed.actions.goToDataStreamsList();
+      });
+      testBed.component.update();
+    });
+
+    test('listed in the table with Managed label', () => {
+      const { table } = testBed;
+      const { tableCellsValues } = table.getMetaData('dataStreamTable');
+
+      expect(tableCellsValues).toEqual([
+        ['', `managed-data-stream${nonBreakingSpace}Managed`, 'green', '1', 'Delete'],
+        ['', 'non-managed-data-stream', 'green', '1', 'Delete'],
+      ]);
+    });
+
+    test('turning off "Include managed" switch hides managed data streams', async () => {
+      const { exists, actions, component, table } = testBed;
+      let { tableCellsValues } = table.getMetaData('dataStreamTable');
+
+      expect(tableCellsValues).toEqual([
+        ['', `managed-data-stream${nonBreakingSpace}Managed`, 'green', '1', 'Delete'],
+        ['', 'non-managed-data-stream', 'green', '1', 'Delete'],
+      ]);
+
+      expect(exists('includeManagedSwitch')).toBe(true);
+
+      await act(async () => {
+        actions.clickIncludeManagedSwitch();
+      });
+      component.update();
+
+      ({ tableCellsValues } = table.getMetaData('dataStreamTable'));
+      expect(tableCellsValues).toEqual([['', 'non-managed-data-stream', 'green', '1', 'Delete']]);
+    });
+  });
 });

x-pack/plugins/index_management/__jest__/client_integration/home/indices_tab.test.ts

@@ -74,7 +74,9 @@ describe('<IndexManagementHome />', () => {
       // The detail panel should still appear even if there are no data streams.
       httpRequestsMockHelpers.setLoadDataStreamsResponse([]);
 
-      httpRequestsMockHelpers.setLoadDataStreamResponse(createDataStreamPayload('dataStream1'));
+      httpRequestsMockHelpers.setLoadDataStreamResponse(
+        createDataStreamPayload({ name: 'dataStream1' })
+      );
 
       testBed = await setup({
         history: createMemoryHistory(),

x-pack/plugins/index_management/common/lib/data_stream_serialization.ts

@@ -17,6 +17,7 @@ export function deserializeDataStream(dataStreamFromEs: DataStreamFromEs): DataS
     ilm_policy: ilmPolicyName,
     store_size: storageSize,
     maximum_timestamp: maxTimeStamp,
+    _meta,
   } = dataStreamFromEs;
 
   return {
@@ -35,6 +36,7 @@ export function deserializeDataStream(dataStreamFromEs: DataStreamFromEs): DataS
     ilmPolicyName,
     storageSize,
     maxTimeStamp,
+    _meta,
   };
 }
 

x-pack/plugins/index_management/common/types/data_streams.ts

@@ -10,13 +10,22 @@ interface TimestampFieldFromEs {
 
 type TimestampField = TimestampFieldFromEs;
 
+interface MetaFieldFromEs {
+  managed_by: string;
+  package: any;
+  managed: boolean;
+}
+
+type MetaField = MetaFieldFromEs;
+
 export type HealthFromEs = 'GREEN' | 'YELLOW' | 'RED';
 
 export interface DataStreamFromEs {
   name: string;
   timestamp_field: TimestampFieldFromEs;
   indices: DataStreamIndexFromEs[];
   generation: number;
+  _meta?: MetaFieldFromEs;
   status: HealthFromEs;
   template: string;
   ilm_policy?: string;
@@ -41,6 +50,7 @@ export interface DataStream {
   ilmPolicyName?: string;
   storageSize?: string;
   maxTimeStamp?: number;
+  _meta?: MetaField;
 }
 
 export interface DataStreamIndex {

x-pack/plugins/index_management/public/application/lib/data_streams.tsx

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { DataStream } from '../../../common';
+
+export const isManagedByIngestManager = (dataStream: DataStream): boolean => {
+  return Boolean(dataStream._meta?.managed && dataStream._meta?.managed_by === 'ingest-manager');
+};
+
+export const filterDataStreams = (dataStreams: DataStream[]): DataStream[] => {
+  return dataStreams.filter((dataStream: DataStream) => !isManagedByIngestManager(dataStream));
+};

x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_list.tsx

@@ -32,6 +32,7 @@ import { documentationService } from '../../../services/documentation';
 import { Section } from '../home';
 import { DataStreamTable } from './data_stream_table';
 import { DataStreamDetailPanel } from './data_stream_detail_panel';
+import { filterDataStreams } from '../../../lib/data_streams';
 
 interface MatchParams {
   dataStreamName?: string;
@@ -52,6 +53,7 @@ export const DataStreamList: React.FunctionComponent<RouteComponentProps<MatchPa
   } = useAppContext();
 
   const [isIncludeStatsChecked, setIsIncludeStatsChecked] = useState(false);
+  const [isIncludeManagedChecked, setIsIncludeManagedChecked] = useState(true);
   const { error, isLoading, data: dataStreams, resendRequest: reload } = useLoadDataStreams({
     includeStats: isIncludeStatsChecked,
   });
@@ -147,11 +149,13 @@ export const DataStreamList: React.FunctionComponent<RouteComponentProps<MatchPa
       />
     );
   } else if (Array.isArray(dataStreams) && dataStreams.length > 0) {
+    const filteredDataStreams = isIncludeManagedChecked
+      ? dataStreams
+      : filterDataStreams(dataStreams);
     content = (
       <>
         <EuiFlexGroup alignItems="center" justifyContent="spaceBetween">
           <EuiFlexItem>
-            {/* TODO: Add a switch for toggling on data streams created by Ingest Manager */}
             <EuiText color="subdued">
               <FormattedMessage
                 id="xpack.idxMgmt.dataStreamList.dataStreamsDescription"
@@ -202,6 +206,35 @@ export const DataStreamList: React.FunctionComponent<RouteComponentProps<MatchPa
               </EuiFlexItem>
             </EuiFlexGroup>
           </EuiFlexItem>
+          <EuiFlexItem grow={false}>
+            <EuiFlexGroup gutterSize="s">
+              <EuiFlexItem grow={false}>
+                <EuiSwitch
+                  label={i18n.translate(
+                    'xpack.idxMgmt.dataStreamListControls.includeManagedSwitchLabel',
+                    {
+                      defaultMessage: 'Include managed',
+                    }
+                  )}
+                  checked={isIncludeManagedChecked}
+                  onChange={(e) => setIsIncludeManagedChecked(e.target.checked)}
+                  data-test-subj="includeManagedSwitch"
+                />
+              </EuiFlexItem>
+
+              <EuiFlexItem grow={false}>
+                <EuiIconTip
+                  content={i18n.translate(
+                    'xpack.idxMgmt.dataStreamListControls.includeManagedSwitchToolTip',
+                    {
+                      defaultMessage: 'Display data streams managed by Ingest Manager',
+                    }
+                  )}
+                  position="top"
+                />
+              </EuiFlexItem>
+            </EuiFlexGroup>
+          </EuiFlexItem>
         </EuiFlexGroup>
 
         <EuiSpacer size="l" />
@@ -212,7 +245,7 @@ export const DataStreamList: React.FunctionComponent<RouteComponentProps<MatchPa
               ? `name="${attemptToURIDecode(dataStreamName)}"`
               : ''
           }
-          dataStreams={dataStreams}
+          dataStreams={filteredDataStreams}
           reload={reload}
           history={history as ScopedHistory}
           includeStats={isIncludeStatsChecked}