Merge remote-tracking branch 'upstream/main' into 3wd-final-readonly

elastic · Sep 3, 2024 · e672f3b · e672f3b
2 parents a0e7132 + c594254
commit e672f3b
Show file tree

Hide file tree

Showing 109 changed files with 1,928 additions and 913 deletions.
diff --git a/package.json b/package.json
@@ -1229,7 +1229,7 @@
     "suricata-sid-db": "^1.0.2",
     "swr": "^2.2.5",
     "symbol-observable": "^1.2.0",
-    "tar": "^6.2.1",
+    "tar": "^7.4.3",
     "textarea-caret": "^3.1.0",
     "tinycolor2": "1.4.1",
     "tinygradient": "0.4.3",
@@ -1609,7 +1609,6 @@
     "@types/styled-components": "^5.1.0",
     "@types/supertest": "^6.0.2",
     "@types/tapable": "^1.0.6",
-    "@types/tar": "^6.1.11",
     "@types/testing-library__jest-dom": "^5.14.7",
     "@types/textarea-caret": "^3.0.1",
     "@types/tinycolor2": "^1.4.1",

diff --git a/packages/kbn-dev-utils/src/extract.ts b/packages/kbn-dev-utils/src/extract.ts
@@ -12,7 +12,7 @@ import Path from 'path';
 import { pipeline } from 'stream';
 import { promisify } from 'util';
 
-import Tar from 'tar';
+import * as tar from 'tar';
 import Yauzl, { ZipFile, Entry } from 'yauzl';
 import * as Rx from 'rxjs';
 import { map, mergeMap, takeUntil } from 'rxjs';
@@ -61,7 +61,7 @@ export async function extract({
   await Fs.mkdir(targetDir, { recursive: true });
 
   if (archivePath.endsWith('.tar') || archivePath.endsWith('.tar.gz')) {
-    return await Tar.x({
+    return await tar.extract({
       file: archivePath,
       cwd: targetDir,
       stripComponents,

diff --git a/src/dev/build/lib/fs.ts b/src/dev/build/lib/fs.ts
@@ -19,7 +19,7 @@ import archiver from 'archiver';
 import globby from 'globby';
 import cpy from 'cpy';
 import del from 'del';
-import tar, { ExtractOptions } from 'tar';
+import * as tar from 'tar';
 import { ToolingLog } from '@kbn/tooling-log';
 
 export function assertAbsolute(path: string) {
@@ -199,7 +199,7 @@ export async function getFileHash(path: string, algo: string) {
 export async function untar(
   source: string,
   destination: string,
-  extractOptions: ExtractOptions = {}
+  extractOptions: tar.TarOptionsWithAliasesAsyncNoFile = {}
 ) {
   assertAbsolute(source);
   assertAbsolute(destination);

diff --git a/src/dev/build/lib/integration_tests/fs.test.ts b/src/dev/build/lib/integration_tests/fs.test.ts
@@ -305,11 +305,10 @@ describe('untar()', () => {
     expect(await read(resolve(destination, 'foo_dir/foo/foo.txt'))).toBe('foo\n');
   });
 
-  it('passed thrid argument to Extract class, overriding path with destination', async () => {
+  it('passed thrid argument to Extract class', async () => {
     const destination = resolve(TMP, 'a/b/c');
 
     await untar(FOO_TAR_PATH, destination, {
-      path: '/dev/null',
       strip: 1,
     });
 

diff --git a/...nified_doc_viewer/public/components/doc_viewer_logs_overview/logs_overview_highlights.tsx b/...nified_doc_viewer/public/components/doc_viewer_logs_overview/logs_overview_highlights.tsx
@@ -52,46 +52,50 @@ export function LogsOverviewHighlights({
     value: flattenedDoc[field],
   });
 
+  const shouldRenderHighlight = (field: keyof LogDocumentOverview) => {
+    return Boolean(formattedDoc[field] && flattenedDoc[field]);
+  };
+
   return (
     <>
       {/* Service & Infrastructure highlight */}
       <HighlightSection
         title={serviceInfraAccordionTitle}
         data-test-subj="unifiedDocViewLogsOverviewHighlightSectionServiceInfra"
       >
-        {formattedDoc[fieldConstants.SERVICE_NAME_FIELD] && (
+        {shouldRenderHighlight(fieldConstants.SERVICE_NAME_FIELD) && (
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewService"
             label={serviceLabel}
             fieldMetadata={fieldsMetadata[fieldConstants.SERVICE_NAME_FIELD]}
             {...getHighlightProps(fieldConstants.SERVICE_NAME_FIELD)}
           />
         )}
-        {formattedDoc[fieldConstants.HOST_NAME_FIELD] && (
+        {shouldRenderHighlight(fieldConstants.HOST_NAME_FIELD) && (
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewHostName"
             label={hostNameLabel}
             fieldMetadata={fieldsMetadata[fieldConstants.HOST_NAME_FIELD]}
             {...getHighlightProps(fieldConstants.HOST_NAME_FIELD)}
           />
         )}
-        {formattedDoc[fieldConstants.TRACE_ID_FIELD] && (
+        {shouldRenderHighlight(fieldConstants.TRACE_ID_FIELD) && (
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewTrace"
             label={traceLabel}
             fieldMetadata={fieldsMetadata[fieldConstants.TRACE_ID_FIELD]}
             {...getHighlightProps(fieldConstants.TRACE_ID_FIELD)}
           />
         )}
-        {formattedDoc[fieldConstants.ORCHESTRATOR_CLUSTER_NAME_FIELD] && (
+        {shouldRenderHighlight(fieldConstants.ORCHESTRATOR_CLUSTER_NAME_FIELD) && (
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewClusterName"
             label={orchestratorClusterNameLabel}
             fieldMetadata={fieldsMetadata[fieldConstants.ORCHESTRATOR_CLUSTER_NAME_FIELD]}
             {...getHighlightProps(fieldConstants.ORCHESTRATOR_CLUSTER_NAME_FIELD)}
           />
         )}
-        {formattedDoc[fieldConstants.ORCHESTRATOR_RESOURCE_ID_FIELD] && (
+        {shouldRenderHighlight(fieldConstants.ORCHESTRATOR_RESOURCE_ID_FIELD) && (
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewResourceId"
             label={orchestratorResourceIdLabel}
@@ -105,7 +109,7 @@ export function LogsOverviewHighlights({
         title={cloudAccordionTitle}
         data-test-subj="unifiedDocViewLogsOverviewHighlightSectionCloud"
       >
-        {formattedDoc[fieldConstants.CLOUD_PROVIDER_FIELD] && (
+        {shouldRenderHighlight(fieldConstants.CLOUD_PROVIDER_FIELD) && (
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewCloudProvider"
             label={cloudProviderLabel}
@@ -120,31 +124,31 @@ export function LogsOverviewHighlights({
             {...getHighlightProps(fieldConstants.CLOUD_PROVIDER_FIELD)}
           />
         )}
-        {formattedDoc[fieldConstants.CLOUD_REGION_FIELD] && (
+        {shouldRenderHighlight(fieldConstants.CLOUD_REGION_FIELD) && (
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewCloudRegion"
             label={cloudRegionLabel}
             fieldMetadata={fieldsMetadata[fieldConstants.CLOUD_REGION_FIELD]}
             {...getHighlightProps(fieldConstants.CLOUD_REGION_FIELD)}
           />
         )}
-        {formattedDoc[fieldConstants.CLOUD_AVAILABILITY_ZONE_FIELD] && (
+        {shouldRenderHighlight(fieldConstants.CLOUD_AVAILABILITY_ZONE_FIELD) && (
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewCloudAz"
             label={cloudAvailabilityZoneLabel}
             fieldMetadata={fieldsMetadata[fieldConstants.CLOUD_AVAILABILITY_ZONE_FIELD]}
             {...getHighlightProps(fieldConstants.CLOUD_AVAILABILITY_ZONE_FIELD)}
           />
         )}
-        {formattedDoc[fieldConstants.CLOUD_PROJECT_ID_FIELD] && (
+        {shouldRenderHighlight(fieldConstants.CLOUD_PROJECT_ID_FIELD) && (
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewCloudProjectId"
             label={cloudProjectIdLabel}
             fieldMetadata={fieldsMetadata[fieldConstants.CLOUD_PROJECT_ID_FIELD]}
             {...getHighlightProps(fieldConstants.CLOUD_PROJECT_ID_FIELD)}
           />
         )}
-        {formattedDoc[fieldConstants.CLOUD_INSTANCE_ID_FIELD] && (
+        {shouldRenderHighlight(fieldConstants.CLOUD_INSTANCE_ID_FIELD) && (
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewCloudInstanceId"
             label={cloudInstanceIdLabel}
@@ -158,23 +162,23 @@ export function LogsOverviewHighlights({
         title={otherAccordionTitle}
         data-test-subj="unifiedDocViewLogsOverviewHighlightSectionOther"
       >
-        {formattedDoc[fieldConstants.LOG_FILE_PATH_FIELD] && (
+        {shouldRenderHighlight(fieldConstants.LOG_FILE_PATH_FIELD) && (
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewLogPathFile"
             label={logPathFileLabel}
             fieldMetadata={fieldsMetadata[fieldConstants.LOG_FILE_PATH_FIELD]}
             {...getHighlightProps(fieldConstants.LOG_FILE_PATH_FIELD)}
           />
         )}
-        {formattedDoc[fieldConstants.DATASTREAM_DATASET_FIELD] && (
+        {shouldRenderHighlight(fieldConstants.DATASTREAM_DATASET_FIELD) && (
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewDataset"
             label={datasetLabel}
             fieldMetadata={fieldsMetadata[fieldConstants.DATASTREAM_DATASET_FIELD]}
             {...getHighlightProps(fieldConstants.DATASTREAM_DATASET_FIELD)}
           />
         )}
-        {formattedDoc[fieldConstants.DATASTREAM_NAMESPACE_FIELD] && (
+        {shouldRenderHighlight(fieldConstants.DATASTREAM_NAMESPACE_FIELD) && (
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewNamespace"
             label={namespaceLabel}
@@ -183,7 +187,7 @@ export function LogsOverviewHighlights({
             {...getHighlightProps(fieldConstants.DATASTREAM_NAMESPACE_FIELD)}
           />
         )}
-        {formattedDoc[fieldConstants.AGENT_NAME_FIELD] && (
+        {shouldRenderHighlight(fieldConstants.AGENT_NAME_FIELD) && (
           <HighlightField
             data-test-subj="unifiedDocViewLogsOverviewLogShipper"
             label={shipperLabel}

diff --git a/x-pack/packages/kbn-cloud-security-posture-common/constants.ts b/x-pack/packages/kbn-cloud-security-posture-common/constants.ts
@@ -9,7 +9,7 @@ export const CSPM_POLICY_TEMPLATE = 'cspm';
 export const CDR_LATEST_NATIVE_MISCONFIGURATIONS_INDEX_PATTERN =
   'logs-cloud_security_posture.findings_latest-default';
 export const CDR_LATEST_THIRD_PARTY_MISCONFIGURATIONS_INDEX_PATTERN =
-  'logs-*_latest_misconfigurations_cdr';
+  'security_solution-*.misconfiguration_latest';
 export const CDR_MISCONFIGURATIONS_INDEX_PATTERN = `${CDR_LATEST_NATIVE_MISCONFIGURATIONS_INDEX_PATTERN},${CDR_LATEST_THIRD_PARTY_MISCONFIGURATIONS_INDEX_PATTERN}`;
 export const LATEST_FINDINGS_RETENTION_POLICY = '26h';
 export const MAX_FINDINGS_TO_LOAD = 500;

diff --git a/x-pack/plugins/aiops/public/ui_actions/create_change_point_chart.tsx b/x-pack/plugins/aiops/public/ui_actions/create_change_point_chart.tsx
@@ -40,7 +40,7 @@ export function createAddChangePointChartAction(
       },
     ],
     order: 10,
-    getIconType: () => 'machineLearningApp',
+    getIconType: () => 'changePointDetection',
     getDisplayName: () =>
       i18n.translate('xpack.aiops.embeddableChangePointChartDisplayName', {
         defaultMessage: 'Change point detection',

diff --git a/x-pack/plugins/cloud_security_posture/common/constants.ts b/x-pack/plugins/cloud_security_posture/common/constants.ts
@@ -41,12 +41,6 @@ export const CDR_MISCONFIGURATIONS_DATA_VIEW_NAME = 'Latest Cloud Security Misco
 export const CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX =
   'security_solution_cdr_latest_misconfigurations';
 
-export const CDR_VULNERABILITIES_DATA_VIEW_NAME = 'Latest Cloud Security Vulnerabilities';
-export const CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX =
-  'security_solution_cdr_latest_vulnerabilities';
-export const CDR_VULNERABILITIES_INDEX_PATTERN =
-  'logs-*_latest_vulnerabilities_cdr,logs-cloud_security_posture.vulnerabilities_latest-default';
-
 export const FINDINGS_INDEX_NAME = 'logs-cloud_security_posture.findings';
 export const FINDINGS_INDEX_PATTERN = 'logs-cloud_security_posture.findings-default*';
 export const FINDINGS_INDEX_DEFAULT_NS = 'logs-cloud_security_posture.findings-default';
@@ -59,17 +53,24 @@ export const BENCHMARK_SCORE_INDEX_TEMPLATE_NAME = 'logs-cloud_security_posture.
 export const BENCHMARK_SCORE_INDEX_PATTERN = 'logs-cloud_security_posture.scores-*';
 export const BENCHMARK_SCORE_INDEX_DEFAULT_NS = 'logs-cloud_security_posture.scores-default';
 
+export const CDR_VULNERABILITIES_DATA_VIEW_NAME = 'Latest Cloud Security Vulnerabilities';
+export const CDR_VULNERABILITIES_DATA_VIEW_ID_PREFIX =
+  'security_solution_cdr_latest_vulnerabilities';
+
 export const VULNERABILITIES_INDEX_NAME = 'logs-cloud_security_posture.vulnerabilities';
 export const VULNERABILITIES_INDEX_PATTERN = 'logs-cloud_security_posture.vulnerabilities-default*';
 export const VULNERABILITIES_INDEX_DEFAULT_NS =
   'logs-cloud_security_posture.vulnerabilities-default';
 
 export const LATEST_VULNERABILITIES_INDEX_TEMPLATE_NAME =
   'logs-cloud_security_posture.vulnerabilities_latest';
-export const LATEST_VULNERABILITIES_INDEX_PATTERN =
-  'logs-cloud_security_posture.vulnerabilities_latest*';
-export const LATEST_VULNERABILITIES_INDEX_DEFAULT_NS =
+
+export const CDR_LATEST_NATIVE_VULNERABILITIES_INDEX_PATTERN =
   'logs-cloud_security_posture.vulnerabilities_latest-default';
+export const CDR_LATEST_THIRD_PARTY_VULNERABILITIES_INDEX_PATTERN =
+  'security_solution-*.vulnerability_latest';
+export const CDR_VULNERABILITIES_INDEX_PATTERN = `${CDR_LATEST_THIRD_PARTY_VULNERABILITIES_INDEX_PATTERN},${CDR_LATEST_NATIVE_VULNERABILITIES_INDEX_PATTERN}`;
+
 export const LATEST_VULNERABILITIES_RETENTION_POLICY = '3d';
 
 export const SECURITY_DEFAULT_DATA_VIEW_ID = 'security-solution-default';

diff --git a/...cloud_security_posture/public/pages/vulnerabilities/hooks/use_grouped_vulnerabilities.tsx b/...cloud_security_posture/public/pages/vulnerabilities/hooks/use_grouped_vulnerabilities.tsx
@@ -11,7 +11,7 @@ import { GenericBuckets, GroupingQuery, RootAggregation } from '@kbn/grouping/sr
 import { useQuery } from '@tanstack/react-query';
 import { lastValueFrom } from 'rxjs';
 import { showErrorToast } from '@kbn/cloud-security-posture';
-import { LATEST_VULNERABILITIES_INDEX_PATTERN } from '../../../../common/constants';
+import { CDR_VULNERABILITIES_INDEX_PATTERN } from '../../../../common/constants';
 import { useKibana } from '../../../common/hooks/use_kibana';
 
 // Elasticsearch returns `null` when a sub-aggregation cannot be computed
@@ -56,7 +56,7 @@ export type VulnerabilitiesRootGroupingAggregation =
 
 export const getGroupedVulnerabilitiesQuery = (query: GroupingQuery) => ({
   ...query,
-  index: LATEST_VULNERABILITIES_INDEX_PATTERN,
+  index: CDR_VULNERABILITIES_INDEX_PATTERN,
   size: 0,
 });
 

diff --git a/.../cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx b/.../cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx
@@ -21,7 +21,7 @@ import { FindingsBaseEsQuery, showErrorToast } from '@kbn/cloud-security-posture
 import { VULNERABILITY_FIELDS } from '../../../common/constants';
 import { CspVulnerabilityFinding } from '../../../../common/schemas';
 import {
-  LATEST_VULNERABILITIES_INDEX_PATTERN,
+  CDR_VULNERABILITIES_INDEX_PATTERN,
   LATEST_VULNERABILITIES_RETENTION_POLICY,
 } from '../../../../common/constants';
 import { useKibana } from '../../../common/hooks/use_kibana';
@@ -56,7 +56,7 @@ export const getVulnerabilitiesQuery = (
   { query, sort }: VulnerabilitiesQuery,
   pageParam: number
 ) => ({
-  index: LATEST_VULNERABILITIES_INDEX_PATTERN,
+  index: CDR_VULNERABILITIES_INDEX_PATTERN,
   sort: getMultiFieldsSort(sort),
   size: MAX_FINDINGS_TO_LOAD,
   query: {
@@ -91,7 +91,7 @@ export const useLatestVulnerabilities = (options: VulnerabilitiesQuery) => {
    * the last loaded record to be used as a from parameter to fetch the next chunk of data.
    */
   return useInfiniteQuery(
-    [LATEST_VULNERABILITIES_INDEX_PATTERN, options],
+    [CDR_VULNERABILITIES_INDEX_PATTERN, options],
     async ({ pageParam }) => {
       const {
         rawResponse: { hits },

diff --git a/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/vulnerabilties.test.tsx b/x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/vulnerabilties.test.tsx
@@ -9,7 +9,7 @@ import Chance from 'chance';
 import { Vulnerabilities } from './vulnerabilities';
 import {
   CDR_MISCONFIGURATIONS_DATA_VIEW_ID_PREFIX,
-  LATEST_VULNERABILITIES_INDEX_DEFAULT_NS,
+  CDR_LATEST_NATIVE_VULNERABILITIES_INDEX_PATTERN,
   VULN_MGMT_POLICY_TEMPLATE,
 } from '../../../common/constants';
 import { useCspSetupStatusApi } from '@kbn/cloud-security-posture/src/hooks/use_csp_setup_status_api';
@@ -71,7 +71,9 @@ describe('<Vulnerabilities />', () => {
         status: 'success',
         data: {
           [VULN_MGMT_POLICY_TEMPLATE]: { status: 'not-deployed' },
-          indicesDetails: [{ index: LATEST_VULNERABILITIES_INDEX_DEFAULT_NS, status: 'empty' }],
+          indicesDetails: [
+            { index: CDR_LATEST_NATIVE_VULNERABILITIES_INDEX_PATTERN, status: 'empty' },
+          ],
         },
       })
     );
@@ -97,7 +99,9 @@ describe('<Vulnerabilities />', () => {
         status: 'success',
         data: {
           [VULN_MGMT_POLICY_TEMPLATE]: { status: 'indexing' },
-          indicesDetails: [{ index: LATEST_VULNERABILITIES_INDEX_DEFAULT_NS, status: 'empty' }],
+          indicesDetails: [
+            { index: CDR_LATEST_NATIVE_VULNERABILITIES_INDEX_PATTERN, status: 'empty' },
+          ],
         },
       })
     );
@@ -121,7 +125,9 @@ describe('<Vulnerabilities />', () => {
         status: 'success',
         data: {
           [VULN_MGMT_POLICY_TEMPLATE]: { status: 'index-timeout' },
-          indicesDetails: [{ index: LATEST_VULNERABILITIES_INDEX_DEFAULT_NS, status: 'empty' }],
+          indicesDetails: [
+            { index: CDR_LATEST_NATIVE_VULNERABILITIES_INDEX_PATTERN, status: 'empty' },
+          ],
         },
       })
     );
@@ -144,7 +150,9 @@ describe('<Vulnerabilities />', () => {
         status: 'success',
         data: {
           [VULN_MGMT_POLICY_TEMPLATE]: { status: 'unprivileged' },
-          indicesDetails: [{ index: LATEST_VULNERABILITIES_INDEX_DEFAULT_NS, status: 'empty' }],
+          indicesDetails: [
+            { index: CDR_LATEST_NATIVE_VULNERABILITIES_INDEX_PATTERN, status: 'empty' },
+          ],
         },
       })
     );
@@ -177,7 +185,7 @@ describe('<Vulnerabilities />', () => {
           indicesDetails: [
             { index: 'logs-cloud_security_posture.findings_latest-default', status: 'empty' },
             { index: 'logs-cloud_security_posture.findings-default*', status: 'empty' },
-            { index: LATEST_VULNERABILITIES_INDEX_DEFAULT_NS, status: 'empty' },
+            { index: CDR_LATEST_NATIVE_VULNERABILITIES_INDEX_PATTERN, status: 'empty' },
           ],
         },
       })