[Fleet] EPM support to handle uploaded file paths

x-pack/plugins/fleet/server/routes/epm/handlers.ts

@@ -4,6 +4,8 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 import { TypeOf } from '@kbn/config-schema';
+import mime from 'mime-types';
+import path from 'path';
 import { RequestHandler, ResponseHeaders, KnownHeaders } from 'src/core/server';
 import {
   GetInfoResponse,
@@ -43,6 +45,8 @@ import {
 import { defaultIngestErrorHandler, ingestErrorToResponseOptions } from '../../errors';
 import { splitPkgKey } from '../../services/epm/registry';
 import { licenseService } from '../../services';
+import { getArchiveEntry } from '../../services/epm/archive/cache';
+import { bufferToStream } from '../../services/epm/streams';
 
 export const getCategoriesHandler: RequestHandler<
   undefined,
@@ -102,22 +106,51 @@ export const getFileHandler: RequestHandler<TypeOf<typeof GetFileRequestSchema.p
 ) => {
   try {
     const { pkgName, pkgVersion, filePath } = request.params;
-    const registryResponse = await getFile(`/package/${pkgName}/${pkgVersion}/${filePath}`);
-
-    const headersToProxy: KnownHeaders[] = ['content-type', 'cache-control'];
-    const proxiedHeaders = headersToProxy.reduce((headers, knownHeader) => {
-      const value = registryResponse.headers.get(knownHeader);
-      if (value !== null) {
-        headers[knownHeader] = value;
+    const savedObjectsClient = context.core.savedObjects.client;
+    const savedObject = await getInstallationObject({ savedObjectsClient, pkgName });
+    const pkgInstallSource = savedObject?.attributes.install_source;
+    // TODO: when package storage is available, remove installSource check and check cache and storage, remove registry call
+    if (pkgInstallSource === 'upload' && pkgVersion === savedObject?.attributes.version) {
+      const headerContentType = mime.contentType(path.extname(filePath));
+      if (!headerContentType) {
+        return response.custom({
+          body: `unknown content type for file: ${filePath}`,
+          statusCode: 400,
+        });
       }
-      return headers;
-    }, {} as ResponseHeaders);
+      const archiveFile = getArchiveEntry(`${pkgName}-${pkgVersion}/${filePath}`);
+      if (!archiveFile) {
+        return response.custom({
+          body: `uploaded package file not found: ${filePath}`,
+          statusCode: 404,
+        });
+      }
+      const headers: ResponseHeaders = {
+        'cache-control': 'max-age=10, public',
+        'content-type': headerContentType,
+      };
+      return response.custom({
+        body: bufferToStream(archiveFile),
+        statusCode: 200,
+        headers,
+      });
+    } else {
+      const registryResponse = await getFile(`/package/${pkgName}/${pkgVersion}/${filePath}`);
+      const headersToProxy: KnownHeaders[] = ['content-type', 'cache-control'];
+      const proxiedHeaders = headersToProxy.reduce((headers, knownHeader) => {
+        const value = registryResponse.headers.get(knownHeader);
+        if (value !== null) {
+          headers[knownHeader] = value;
+        }
+        return headers;
+      }, {} as ResponseHeaders);
 
-    return response.custom({
-      body: registryResponse.body,
-      statusCode: registryResponse.status,
-      headers: proxiedHeaders,
-    });
+      return response.custom({
+        body: registryResponse.body,
+        statusCode: registryResponse.status,
+        headers: proxiedHeaders,
+      });
+    }
   } catch (error) {
     return defaultIngestErrorHandler({ error, response });
   }

x-pack/plugins/fleet/server/services/epm/packages/get.ts

@@ -98,7 +98,10 @@ export async function getPackageInfo(options: {
   const getPackageRes = await getPackageFromSource({
     pkgName,
     pkgVersion,
-    pkgInstallSource: savedObject?.attributes.install_source,
+    pkgInstallSource:
+      savedObject?.attributes.version === pkgVersion
+        ? savedObject?.attributes.install_source
+        : 'registry',
   });
   const paths = getPackageRes.paths;
   const packageInfo = getPackageRes.packageInfo;

x-pack/test/fleet_api_integration/apis/epm/file.ts

@@ -4,6 +4,9 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import fs from 'fs';
+import path from 'path';
+import expect from '@kbn/expect';
 import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
 import { warnAndSkipTest } from '../../helpers';
 
@@ -14,79 +17,175 @@ export default function ({ getService }: FtrProviderContext) {
 
   const server = dockerServers.get('registry');
   describe('EPM - package file', () => {
-    it('fetches a .png screenshot image', async function () {
-      if (server.enabled) {
-        await supertest
-          .get('/api/fleet/epm/packages/filetest/0.1.0/img/screenshots/metricbeat_dashboard.png')
-          .set('kbn-xsrf', 'xxx')
-          .expect('Content-Type', 'image/png')
-          .expect(200);
-      } else {
-        warnAndSkipTest(this, log);
-      }
-    });
+    describe('it gets files from registry', () => {
+      it('fetches a .png screenshot image', async function () {
+        if (server.enabled) {
+          const res = await supertest
+            .get('/api/fleet/epm/packages/filetest/0.1.0/img/screenshots/metricbeat_dashboard.png')
+            .set('kbn-xsrf', 'xxx')
+            .expect('Content-Type', 'image/png')
+            .expect(200);
+          expect(Buffer.isBuffer(res.body)).to.equal(true);
+        } else {
+          warnAndSkipTest(this, log);
+        }
+      });
 
-    it('fetches an .svg icon image', async function () {
-      if (server.enabled) {
-        await supertest
-          .get('/api/fleet/epm/packages/filetest/0.1.0/img/logo.svg')
-          .set('kbn-xsrf', 'xxx')
-          .expect('Content-Type', 'image/svg+xml')
-          .expect(200);
-      } else {
-        warnAndSkipTest(this, log);
-      }
-    });
+      it('fetches an .svg icon image', async function () {
+        if (server.enabled) {
+          const res = await supertest
+            .get('/api/fleet/epm/packages/filetest/0.1.0/img/logo.svg')
+            .set('kbn-xsrf', 'xxx')
+            .expect('Content-Type', 'image/svg+xml')
+            .expect(200);
+          expect(Buffer.isBuffer(res.body)).to.equal(true);
+        } else {
+          warnAndSkipTest(this, log);
+        }
+      });
 
-    it('fetches a .json kibana visualization file', async function () {
-      if (server.enabled) {
-        await supertest
-          .get(
-            '/api/fleet/epm/packages/filetest/0.1.0/kibana/visualization/sample_visualization.json'
-          )
-          .set('kbn-xsrf', 'xxx')
-          .expect('Content-Type', 'application/json; charset=utf-8')
-          .expect(200);
-      } else {
-        warnAndSkipTest(this, log);
-      }
-    });
+      it('fetches a .json kibana visualization file', async function () {
+        if (server.enabled) {
+          const res = await supertest
+            .get(
+              '/api/fleet/epm/packages/filetest/0.1.0/kibana/visualization/sample_visualization.json'
+            )
+            .set('kbn-xsrf', 'xxx')
+            .expect('Content-Type', 'application/json; charset=utf-8')
+            .expect(200);
+          expect(typeof res.body).to.equal('object');
+        } else {
+          warnAndSkipTest(this, log);
+        }
+      });
 
-    it('fetches a .json kibana dashboard file', async function () {
-      if (server.enabled) {
-        await supertest
-          .get('/api/fleet/epm/packages/filetest/0.1.0/kibana/dashboard/sample_dashboard.json')
-          .set('kbn-xsrf', 'xxx')
-          .expect('Content-Type', 'application/json; charset=utf-8')
-          .expect(200);
-      } else {
-        warnAndSkipTest(this, log);
-      }
-    });
+      it('fetches a .json kibana dashboard file', async function () {
+        if (server.enabled) {
+          const res = await supertest
+            .get('/api/fleet/epm/packages/filetest/0.1.0/kibana/dashboard/sample_dashboard.json')
+            .set('kbn-xsrf', 'xxx')
+            .expect('Content-Type', 'application/json; charset=utf-8')
+            .expect(200);
+          expect(typeof res.body).to.equal('object');
+        } else {
+          warnAndSkipTest(this, log);
+        }
+      });
 
-    it('fetches a .json search file', async function () {
-      if (server.enabled) {
+      it('fetches a .json search file', async function () {
+        if (server.enabled) {
+          const res = await supertest
+            .get('/api/fleet/epm/packages/filetest/0.1.0/kibana/search/sample_search.json')
+            .set('kbn-xsrf', 'xxx')
+            .expect('Content-Type', 'application/json; charset=utf-8')
+            .expect(200);
+          expect(typeof res.body).to.equal('object');
+        } else {
+          warnAndSkipTest(this, log);
+        }
+      });
+    });
+    describe('it gets files from an uploaded package', () => {
+      before(async () => {
+        if (!server.enabled) return;
+        const testPkgArchiveTgz = path.join(
+          path.dirname(__filename),
+          '../fixtures/direct_upload_packages/apache_0.1.4.tar.gz'
+        );
+        const buf = fs.readFileSync(testPkgArchiveTgz);
         await supertest
-          .get('/api/fleet/epm/packages/filetest/0.1.0/kibana/search/sample_search.json')
-          .set('kbn-xsrf', 'xxx')
-          .expect('Content-Type', 'application/json; charset=utf-8')
+          .post(`/api/fleet/epm/packages`)
+          .set('kbn-xsrf', 'xxxx')
+          .type('application/gzip')
+          .send(buf)
           .expect(200);
-      } else {
-        warnAndSkipTest(this, log);
-      }
+      });
+      after(async () => {
+        if (!server.enabled) return;
+        await supertest.delete(`/api/fleet/epm/packages/apache-0.1.4`).set('kbn-xsrf', 'xxxx');
+      });
+      it('fetches a .png screenshot image', async function () {
+        if (server.enabled) {
+          const res = await supertest
+            .get('/api/fleet/epm/packages/apache/0.1.4/img/kibana-apache-test.png')
+            .set('kbn-xsrf', 'xxx')
+            .expect('Content-Type', 'image/png')
+            .expect(200);
+          expect(Buffer.isBuffer(res.body)).to.equal(true);
+        } else {
+          warnAndSkipTest(this, log);
+        }
+      });
+      it('fetches the logo', async function () {
+        if (server.enabled) {
+          const res = await supertest
+            .get('/api/fleet/epm/packages/apache/0.1.4/img/logo_apache_test.svg')
+            .set('kbn-xsrf', 'xxx')
+            .expect('Content-Type', 'image/svg+xml')
+            .expect(200);
+          await supertest
+            .get('/api/fleet/epm/packages/apache/0.1.4/img/logo_apache.svg')
+            .set('kbn-xsrf', 'xxx')
+            .expect(404);
+          expect(Buffer.isBuffer(res.body)).to.equal(true);
+        } else {
+          warnAndSkipTest(this, log);
+        }
+      });
+
+      it('fetches a .json kibana dashboard file', async function () {
+        if (server.enabled) {
+          const res = await supertest
+            .get(
+              '/api/fleet/epm/packages/apache/0.1.4/kibana/dashboard/apache-Logs-Apache-Dashboard-ecs-new.json'
+            )
+            .set('kbn-xsrf', 'xxx')
+            .expect('Content-Type', 'application/json; charset=utf-8')
+            .expect(200);
+          expect(typeof res.body).to.equal('object');
+        } else {
+          warnAndSkipTest(this, log);
+        }
+      });
+
+      it('fetches a README file', async function () {
+        if (server.enabled) {
+          const res = await supertest
+            .get('/api/fleet/epm/packages/apache/0.1.4/docs/README.md')
+            .set('kbn-xsrf', 'xxx')
+            .expect('Content-Type', 'text/markdown; charset=utf-8')
+            .expect(200);
+          expect(res.text).to.equal('# Apache Uploaded Test Integration');
+        } else {
+          warnAndSkipTest(this, log);
+        }
+      });
+
+      it('fetches the logo of a not uploaded (and installed) version from the registry when another version is uploaded (and installed)', async function () {
+        if (server.enabled) {
+          const res = await supertest
+            .get('/api/fleet/epm/packages/apache/0.1.3/img/logo_apache.svg')
+            .set('kbn-xsrf', 'xxx')
+            .expect('Content-Type', 'image/svg+xml')
+            .expect(200);
+          expect(Buffer.isBuffer(res.body)).to.equal(true);
+        } else {
+          warnAndSkipTest(this, log);
+        }
+      });
     });
-  });
 
-  // Disabled for now as we don't serve prebuilt index patterns in current packages.
-  // it('fetches an .json index pattern file', async function () {
-  //   if (server.enabled) {
-  //     await supertest
-  //       .get('/api/fleet/epm/packages/filetest/0.1.0/kibana/index-pattern/sample-*.json')
-  //       .set('kbn-xsrf', 'xxx')
-  //       .expect('Content-Type', 'application/json; charset=utf-8')
-  //       .expect(200);
-  //   } else {
-  //     warnAndSkipTest(this, log);
-  //   }
-  // });
+    // Disabled for now as we don't serve prebuilt index patterns in current packages.
+    // it('fetches an .json index pattern file', async function () {
+    //   if (server.enabled) {
+    //     await supertest
+    //       .get('/api/fleet/epm/packages/filetest/0.1.0/kibana/index-pattern/sample-*.json')
+    //       .set('kbn-xsrf', 'xxx')
+    //       .expect('Content-Type', 'application/json; charset=utf-8')
+    //       .expect(200);
+    //   } else {
+    //     warnAndSkipTest(this, log);
+    //   }
+    // });
+  });
 }

x-pack/test/fleet_api_integration/apis/epm/get.ts

@@ -71,6 +71,25 @@ export default function (providerContext: FtrProviderContext) {
         warnAndSkipTest(this, log);
       }
     });
+    it('returns correct package info from registry if a different version is installed by upload', async function () {
+      if (server.enabled) {
+        const buf = fs.readFileSync(testPkgArchiveZip);
+        await supertest
+          .post(`/api/fleet/epm/packages`)
+          .set('kbn-xsrf', 'xxxx')
+          .type('application/zip')
+          .send(buf)
+          .expect(200);
+
+        const res = await supertest.get(`/api/fleet/epm/packages/apache-0.1.3`).expect(200);
+        const packageInfo = res.body.response;
+        expect(packageInfo.description).to.equal('Apache Integration');
+        expect(packageInfo.download).to.not.equal(undefined);
+        await uninstallPackage(testPkgKey);
+      } else {
+        warnAndSkipTest(this, log);
+      }
+    });
     it('returns a 500 for a package key without a proper name', async function () {
       if (server.enabled) {
         await supertest.get('/api/fleet/epm/packages/-0.1.0').expect(500);