[Fleet] Add APM instrumentation for package install process

x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts

@@ -42,11 +42,11 @@ import { packagePolicyService } from '../..';
 
 import { createInstallation, saveKibanaAssetsRefs, updateVersion } from './install';
 import { deleteKibanaSavedObjectsAssets } from './remove';
+import { withPackageSpan } from './utils';
 
 // this is only exported for testing
 // use a leading underscore to indicate it's not the supported path
 // only the more explicit `installPackage*` functions should be used
-
 export async function _installPackage({
   savedObjectsClient,
   savedObjectsImporter,
@@ -106,60 +106,46 @@ export async function _installPackage({
       });
     }
 
-    const kibanaAssets = await getKibanaAssets(paths);
-    if (installedPkg) await deleteKibanaSavedObjectsAssets({ savedObjectsClient, installedPkg });
-    // save new kibana refs before installing the assets
-    const installedKibanaAssetsRefs = await saveKibanaAssetsRefs(
-      savedObjectsClient,
-      pkgName,
-      kibanaAssets
-    );
+    const installedKibanaAssetsRefs = await withPackageSpan('Install Kibana assets', async () => {
+      const kibanaAssets = await getKibanaAssets(paths);
+      if (installedPkg) await deleteKibanaSavedObjectsAssets({ savedObjectsClient, installedPkg });
+      // save new kibana refs before installing the assets
+      const assetRefs = await saveKibanaAssetsRefs(savedObjectsClient, pkgName, kibanaAssets);
+
+      await installKibanaAssets({
+        logger,
+        savedObjectsImporter,
+        pkgName,
+        kibanaAssets,
+      });
 
-    await installKibanaAssets({
-      logger,
-      savedObjectsImporter,
-      pkgName,
-      kibanaAssets,
+      return assetRefs;
     });
 
     // the rest of the installation must happen in sequential order
     // currently only the base package has an ILM policy
     // at some point ILM policies can be installed/modified
     // per data stream and we should then save them
-    await installILMPolicy(packageInfo, paths, esClient, logger);
+    await withPackageSpan('Install ILM policies', () =>
+      installILMPolicy(packageInfo, paths, esClient, logger)
+    );
 
-    const installedDataStreamIlm = await installIlmForDataStream(
-      packageInfo,
-      paths,
-      esClient,
-      savedObjectsClient,
-      logger
+    const installedDataStreamIlm = await withPackageSpan('Install Data Stream ILM policies', () =>
+      installIlmForDataStream(packageInfo, paths, esClient, savedObjectsClient, logger)
     );
 
     // installs ml models
-    const installedMlModel = await installMlModel(
-      packageInfo,
-      paths,
-      esClient,
-      savedObjectsClient,
-      logger
+    const installedMlModel = await withPackageSpan('Install ML models', () =>
+      installMlModel(packageInfo, paths, esClient, savedObjectsClient, logger)
     );
 
     // installs versionized pipelines without removing currently installed ones
-    const installedPipelines = await installPipelines(
-      packageInfo,
-      paths,
-      esClient,
-      savedObjectsClient,
-      logger
+    const installedPipelines = await withPackageSpan('Install ingest pipelines', () =>
+      installPipelines(packageInfo, paths, esClient, savedObjectsClient, logger)
     );
     // install or update the templates referencing the newly installed pipelines
-    const installedTemplates = await installTemplates(
-      packageInfo,
-      esClient,
-      logger,
-      paths,
-      savedObjectsClient
+    const installedTemplates = await withPackageSpan('Install index templates', () =>
+      installTemplates(packageInfo, esClient, logger, paths, savedObjectsClient)
     );
 
     try {
@@ -169,14 +155,12 @@ export async function _installPackage({
     }
 
     // update current backing indices of each data stream
-    await updateCurrentWriteIndices(esClient, logger, installedTemplates);
+    await withPackageSpan('Update write indices', () =>
+      updateCurrentWriteIndices(esClient, logger, installedTemplates)
+    );
 
-    const installedTransforms = await installTransform(
-      packageInfo,
-      paths,
-      esClient,
-      savedObjectsClient,
-      logger
+    const installedTransforms = await withPackageSpan('Install transforms', () =>
+      installTransform(packageInfo, paths, esClient, savedObjectsClient, logger)
     );
 
     // If this is an update or retrying an update, delete the previous version's pipelines
@@ -187,30 +171,36 @@ export async function _installPackage({
       (installType === 'update' || installType === 'reupdate') &&
       installedPkg
     ) {
-      await deletePreviousPipelines(
-        esClient,
-        savedObjectsClient,
-        pkgName,
-        installedPkg.attributes.version
+      await withPackageSpan('Delete previous ingest pipelines', () =>
+        deletePreviousPipelines(
+          esClient,
+          savedObjectsClient,
+          pkgName,
+          installedPkg.attributes.version
+        )
       );
     }
     // pipelines from a different version may have installed during a failed update
     if (installType === 'rollback' && installedPkg) {
-      await deletePreviousPipelines(
-        esClient,
-        savedObjectsClient,
-        pkgName,
-        installedPkg.attributes.install_version
+      await await withPackageSpan('Delete previous ingest pipelines', () =>
+        deletePreviousPipelines(
+          esClient,
+          savedObjectsClient,
+          pkgName,
+          installedPkg.attributes.install_version
+        )
       );
     }
     const installedTemplateRefs = getAllTemplateRefs(installedTemplates);
 
-    const packageAssetResults = await saveArchiveEntries({
-      savedObjectsClient,
-      paths,
-      packageInfo,
-      installSource,
-    });
+    const packageAssetResults = await withPackageSpan('Update archive entries', () =>
+      saveArchiveEntries({
+        savedObjectsClient,
+        paths,
+        packageInfo,
+        installSource,
+      })
+    );
     const packageAssetRefs: PackageAssetReference[] = packageAssetResults.saved_objects.map(
       (result) => ({
         id: result.id,
@@ -221,26 +211,26 @@ export async function _installPackage({
     // update to newly installed version when all assets are successfully installed
     if (installedPkg) await updateVersion(savedObjectsClient, pkgName, pkgVersion);
 
-    const updatedPackage = await savedObjectsClient.update<Installation>(
-      PACKAGES_SAVED_OBJECT_TYPE,
-      pkgName,
-      {
+    const updatedPackage = await withPackageSpan('Update install status', () =>
+      savedObjectsClient.update<Installation>(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
         install_version: pkgVersion,
         install_status: 'installed',
         package_assets: packageAssetRefs,
-      }
+      })
     );
 
     // If the package is flagged with the `keep_policies_up_to_date` flag, upgrade its
     // associated package policies after installation
     if (updatedPackage.attributes.keep_policies_up_to_date) {
-      const policyIdsToUpgrade = await packagePolicyService.listIds(savedObjectsClient, {
-        page: 1,
-        perPage: SO_SEARCH_LIMIT,
-        kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:${pkgName}`,
-      });
+      await withPackageSpan('Upgrade package policies', async () => {
+        const policyIdsToUpgrade = await packagePolicyService.listIds(savedObjectsClient, {
+          page: 1,
+          perPage: SO_SEARCH_LIMIT,
+          kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:${pkgName}`,
+        });
 
-      await packagePolicyService.upgrade(savedObjectsClient, esClient, policyIdsToUpgrade.items);
+        await packagePolicyService.upgrade(savedObjectsClient, esClient, policyIdsToUpgrade.items);
+      });
     }
 
     return [

x-pack/plugins/fleet/server/services/epm/packages/install.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import apm from 'elastic-apm-node';
 import { i18n } from '@kbn/i18n';
 import semverLt from 'semver/functions/lt';
 import type Boom from '@hapi/boom';
@@ -250,6 +251,10 @@ async function installPackageFromRegistry({
   // TODO: change epm API to /packageName/version so we don't need to do this
   const { pkgName, pkgVersion } = Registry.splitPkgKey(pkgkey);
 
+  // Workaround apm issue with async spans: https://github.com/elastic/apm-agent-nodejs/issues/2611
+  await Promise.resolve();
+  const span = apm.startSpan(`Install package from registry ${pkgName}@${pkgVersion}`, 'package');
+
   // if an error happens during getInstallType, report that we don't know
   let installType: InstallType = 'unknown';
 
@@ -260,6 +265,12 @@ async function installPackageFromRegistry({
     const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName });
     installType = getInstallType({ pkgVersion, installedPkg });
 
+    span?.addLabels({
+      packageName: pkgName,
+      packageVersion: pkgVersion,
+      installType,
+    });
+
     // get latest package version
     const latestPackage = await Registry.fetchFindLatestPackageOrThrow(pkgName, {
       ignoreConstraints,
@@ -326,7 +337,7 @@ async function installPackageFromRegistry({
 
     // try installing the package, if there was an error, call error handler and rethrow
     // @ts-expect-error status is string instead of InstallResult.status 'installed' | 'already_installed'
-    return _installPackage({
+    return await _installPackage({
       savedObjectsClient,
       savedObjectsImporter,
       esClient,
@@ -377,6 +388,8 @@ async function installPackageFromRegistry({
       installType,
       installSource: 'registry',
     };
+  } finally {
+    span?.end();
   }
 }
 
@@ -395,6 +408,10 @@ async function installPackageByUpload({
   contentType,
   spaceId,
 }: InstallUploadedArchiveParams): Promise<InstallResult> {
+  // Workaround apm issue with async spans: https://github.com/elastic/apm-agent-nodejs/issues/2611
+  await Promise.resolve();
+  const span = apm.startSpan(`Install package from upload`, 'package');
+
   const logger = appContextService.getLogger();
   // if an error happens during getInstallType, report that we don't know
   let installType: InstallType = 'unknown';
@@ -409,6 +426,12 @@ async function installPackageByUpload({
 
     installType = getInstallType({ pkgVersion: packageInfo.version, installedPkg });
 
+    span?.addLabels({
+      packageName: packageInfo.name,
+      packageVersion: packageInfo.version,
+      installType,
+    });
+
     telemetryEvent.packageName = packageInfo.name;
     telemetryEvent.newVersion = packageInfo.version;
     telemetryEvent.installType = installType;
@@ -434,7 +457,7 @@ async function installPackageByUpload({
       .createImporter(savedObjectsClient);
 
     // @ts-expect-error status is string instead of InstallResult.status 'installed' | 'already_installed'
-    return _installPackage({
+    return await _installPackage({
       savedObjectsClient,
       savedObjectsImporter,
       esClient,
@@ -466,6 +489,8 @@ async function installPackageByUpload({
       errorMessage: e.message,
     });
     return { error: e, installType, installSource: 'upload' };
+  } finally {
+    span?.end();
   }
 }
 

x-pack/plugins/fleet/server/services/epm/packages/utils.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { withSpan } from '@kbn/apm-utils';
+
+export const withPackageSpan = <T>(stepName: string, func: () => Promise<T>) =>
+  withSpan({ name: stepName, type: 'package' }, func);