Plugin zip support

package.json

@@ -50,6 +50,7 @@
     "url": "https://github.com/elastic/kibana.git"
   },
   "dependencies": {
+    "@bigfunger/decompress-zip": "^0.2.0-stripfix2",
     "@spalger/angular-bootstrap": "0.10.0",
     "@spalger/angular-nvd3": "1.0.0-beta",
     "@spalger/filesaver": "1.1.2",
@@ -159,15 +160,15 @@
     "mocha": "2.3.0",
     "nock": "2.10.0",
     "Nonsense": "0.1.2",
-    "npm": "3.2.2",
+    "npm": "2.11.0",
     "portscanner": "1.0.0",
     "simple-git": "1.8.0",
     "sinon": "1.16.1",
     "source-map": "0.4.4",
     "wreck": "6.2.0"
   },
   "engines": {
-    "node": "2.5",
-    "npm": "3.2"
+    "node": "0.12",
+    "npm": "2.14.3"
   }
 }

src/cli/plugin/__tests__/pluginDownloader.js

@@ -82,25 +82,6 @@ describe('kibana cli', function () {
         });
       });
 
-      it('should abort the download and extraction for a corrupt archive.', function () {
-        var filename = join(__dirname, 'replies/corrupt.tar.gz');
-        var couchdb = nock('http://www.files.com')
-        .get('/plugin.tar.gz')
-        .replyWithFile(200, filename);
-
-        var source = 'http://www.files.com/plugin.tar.gz';
-
-        var errorStub = sinon.stub();
-        return downloader._downloadSingle(source, testWorkingPath, 0, logger)
-        .catch(errorStub)
-        .then(function (data) {
-          expect(errorStub.called).to.be(true);
-
-          var files = glob.sync('**/*', { cwd: testWorkingPath });
-          expect(files).to.eql([]);
-        });
-      });
-
     });
 
     describe('download', function () {

src/cli/plugin/__tests__/progressReporter.js

@@ -139,7 +139,7 @@ describe('kibana cli', function () {
           .catch(errorStub)
           .then(function (data) {
             expect(logger.log.callCount).to.be(2);
-            expect(logger.log.getCall(0).args[0]).to.match(/downloading/i);
+            expect(logger.log.getCall(0).args[0]).to.match(/transfer/i);
             expect(logger.log.getCall(1).args[0]).to.match(/complete/i);
           });
         });
@@ -154,7 +154,7 @@ describe('kibana cli', function () {
           .catch(errorStub)
           .then(function (data) {
             expect(logger.log.callCount).to.be(22);
-            expect(logger.log.getCall(0).args[0]).to.match(/downloading/i);
+            expect(logger.log.getCall(0).args[0]).to.match(/transfer/i);
             expect(logger.log.getCall(1).args[0]).to.be('.');
             expect(logger.log.getCall(2).args[0]).to.be('.');
             expect(logger.log.getCall(3).args[0]).to.be('.');
@@ -211,7 +211,7 @@ describe('kibana cli', function () {
           .catch(errorStub)
           .then(function (data) {
             expect(errorStub.called).to.be(false);
-            expect(logger.log.getCall(0).args[0]).to.match(/downloading/i);
+            expect(logger.log.getCall(0).args[0]).to.match(/transfer/i);
             expect(logger.log.getCall(21).args[0]).to.match(/complete/i);
           });
         });

src/cli/plugin/extractors/tarGz.js

@@ -0,0 +1,30 @@
+var zlib = require('zlib');
+var Promise = require('bluebird');
+var fs = require('fs');
+var tar = require('tar');
+
+module.exports = function (settings, logger) {
+  return new Promise(function (resolve, reject) {
+    var gunzip = zlib.createGunzip();
+    var tarExtract = new tar.Extract({ path: settings.workingPath, strip: 1 });
+
+    logger.log('Extracting plugin archive');
+
+    fs.createReadStream(settings.tempArchiveFile)
+    .pipe(gunzip)
+    .on('error', handleError)
+    .pipe(tarExtract)
+    .on('error', handleError)
+    .on('end', handleEnd);
+
+    function handleError(err) {
+      logger.error(err);
+      return reject(new Error('Error extracting plugin archive'));
+    }
+
+    function handleEnd() {
+      logger.log('Extraction complete');
+      return resolve();
+    }
+  });
+};

src/cli/plugin/extractors/zip.js

@@ -0,0 +1,28 @@
+var Promise = require('bluebird');
+var DecompressZip = require('@bigfunger/decompress-zip');
+
+module.exports = function (settings, logger) {
+  return new Promise(function (resolve, reject) {
+    logger.log('Extracting plugin archive');
+
+    var unzipper = new DecompressZip(settings.tempArchiveFile);
+
+    unzipper.on('error', function (err) {
+      logger.error(err);
+      return reject(new Error('Error extracting plugin archive'));
+    });
+
+    unzipper.on('extract', function (log) {
+      logger.log('Extraction complete');
+      return resolve();
+    });
+
+    unzipper.extract({
+      path: settings.workingPath,
+      filter: function (file) {
+        return file.type !== 'SymbolicLink';
+      },
+      strip: 1
+    });
+  });
+};

src/cli/plugin/pluginDownloader.js

@@ -1,39 +1,35 @@
-var _ = require('lodash');
-var zlib = require('zlib');
-var Promise = require('bluebird');
-var url = require('url');
-var fs = require('fs');
-var request = require('request');
-var tar = require('tar');
-var progressReporter = require('./progressReporter');
+let _ = require('lodash');
+let Promise = require('bluebird');
+let urlParse = require('url').parse;
+let fs = require('fs');
+let request = require('request');
+let progressReporter = require('./progressReporter');
 
 module.exports = function (settings, logger) {
+  let archiveType;
 
   //Attempts to download each url in turn until one is successful
   function download() {
-    var urls = settings.urls;
+    let urls = settings.urls;
 
     function tryNext() {
-      var sourceUrl = urls.shift();
+      let sourceUrl = urls.shift();
       if (!sourceUrl) {
-        throw new Error('Not a valid url.');
+        throw new Error('No valid url specified.');
       }
 
-      logger.log('Attempting to extract from ' + sourceUrl);
+      logger.log(`Attempting to transfer from ${sourceUrl}`);
 
-      return Promise.try(function () {
+      return Promise.try(() => {
         return downloadSingle(sourceUrl, settings.workingPath, settings.timeout, logger)
-        .catch(function (err) {
+        .catch((err) => {
           if (err.message === 'ENOTFOUND') {
             return tryNext();
           }
-          if (err.message === 'EEXTRACT') {
-            throw (new Error('Error extracting the plugin archive... is this a valid tar.gz file?'));
-          }
           throw (err);
         });
       })
-      .catch(function (err) {
+      .catch((err) => {
         //Special case for when request.get throws an exception
         if (err.message.match(/invalid uri/i)) {
           return tryNext();
@@ -45,51 +41,69 @@ module.exports = function (settings, logger) {
     return tryNext();
   }
 
-  //Attempts to download a single url
   function downloadSingle(source, dest, timeout) {
-    var gunzip = zlib.createGunzip();
-    var tarExtract = new tar.Extract({ path: dest, strip: 1 });
-
-    var requestOptions = { url: source };
+    let requestOptions = { url: source };
     if (timeout !== 0) {
       requestOptions.timeout = timeout;
     }
 
-    return wrappedRequest(requestOptions)
-    .then(function (fileStream) {
-      var reporter = progressReporter(logger, fileStream);
+    return getReadStream(requestOptions)
+    .then((readStream) => {
+      let writeStream = fs.createWriteStream(settings.tempArchiveFile);
+      let reporter = progressReporter(logger, readStream);
 
-      fileStream
+      readStream
       .on('response', reporter.handleResponse)
       .on('data', reporter.handleData)
       .on('error', _.partial(reporter.handleError, 'ENOTFOUND'))
-      .pipe(gunzip)
-      .on('error', _.partial(reporter.handleError, 'EEXTRACT'))
-      .pipe(tarExtract)
-      .on('error', _.partial(reporter.handleError, 'EEXTRACT'))
-      .on('end', reporter.handleEnd);
+      .pipe(writeStream)
+      .on('finish', reporter.handleEnd);
 
       return reporter.promise;
+    })
+    .then((downloadResult) => {
+      archiveType = archiveType || downloadResult.archiveType;
+      return archiveType;
     });
   }
 
-  function wrappedRequest(requestOptions) {
-    return Promise.try(function () {
-      let urlInfo = url.parse(requestOptions.url);
-      if (/^file/.test(urlInfo.protocol)) {
-        return fs.createReadStream(urlInfo.path);
-      } else {
-        return request.get(requestOptions);
-      }
+  function getReadStream(requestOptions) {
+    let urlInfo = urlParse(requestOptions.url);
+
+    if (/^file/.test(urlInfo.protocol)) {
+      return getReadStreamFromFile(urlInfo.path);
+    } else {
+      return getReadStreamFromUrl(requestOptions);
+    }
+  }
+
+  function getReadStreamFromFile(filePath) {
+    return Promise.try(() => {
+      archiveType = getArchiveTypeFromFilename(filePath);
+      return fs.createReadStream(filePath);
+    });
+  }
+
+  function getReadStreamFromUrl(requestOptions) {
+    return Promise.try(() => {
+      return request.get(requestOptions);
     })
-    .catch(function (err) {
+    .catch((err) => {
       if (err.message.match(/invalid uri/i)) {
         throw new Error('ENOTFOUND');
       }
       throw err;
     });
   }
 
+  function getArchiveTypeFromFilename(filePath) {
+    if (/\.zip$/i.test(filePath)) {
+      return '.zip';
+    }
+    if (/\.tar\.gz$/i.test(filePath)) {
+      return '.tar.gz';
+    }
+  }
 
   return {
     download: download,

src/cli/plugin/pluginExtractor.js

@@ -0,0 +1,15 @@
+var zipExtract = require('./extractors/zip');
+var tarGzExtract = require('./extractors/tarGz');
+
+module.exports = function (settings, logger, archiveType) {
+  switch (archiveType) {
+    case '.zip':
+      return zipExtract(settings, logger);
+      break;
+    case '.tar.gz':
+      return tarGzExtract(settings, logger);
+      break;
+    default:
+      throw new Error('Unsupported archive format.');
+  }
+};

src/cli/plugin/pluginInstaller.js

@@ -1,11 +1,15 @@
 let _ = require('lodash');
-var utils = require('requirefrom')('src/utils');
-var fromRoot = utils('fromRoot');
-var pluginDownloader = require('./pluginDownloader');
-var pluginCleaner = require('./pluginCleaner');
-var KbnServer = require('../../server/KbnServer');
-var readYamlConfig = require('../serve/readYamlConfig');
-var fs = require('fs');
+let utils = require('requirefrom')('src/utils');
+let fromRoot = utils('fromRoot');
+let pluginDownloader = require('./pluginDownloader');
+let pluginCleaner = require('./pluginCleaner');
+let pluginExtractor = require('./pluginExtractor');
+let KbnServer = require('../../server/KbnServer');
+let readYamlConfig = require('../serve/readYamlConfig');
+let fs = require('fs');
+let Promise = require('bluebird');
+let rimraf = require('rimraf');
+let mkdirp = Promise.promisify(require('mkdirp'));
 
 module.exports = {
   install: install
@@ -23,13 +27,19 @@ function install(settings, logger) {
     if (e.code !== 'ENOENT') throw e;
   }
 
-  var cleaner = pluginCleaner(settings, logger);
-  var downloader = pluginDownloader(settings, logger);
+  let cleaner = pluginCleaner(settings, logger);
+  let downloader = pluginDownloader(settings, logger);
 
   return cleaner.cleanPrevious()
-  .then(function () {
+  .then(() => {
+    return mkdirp(settings.workingPath);
+  })
+  .then(() => {
     return downloader.download();
   })
+  .then((archiveType) => {
+    return pluginExtractor (settings, logger, archiveType);
+  })
   .then(async function() {
     logger.log('Optimizing and caching browser bundles...');
     let serverConfig = _.merge(
@@ -59,12 +69,15 @@ function install(settings, logger) {
     await kbnServer.ready();
     await kbnServer.close();
   })
-  .then(function () {
+  .then(() => {
+    rimraf.sync(settings.tempArchiveFile);
+  })
+  .then(() => {
     fs.renameSync(settings.workingPath, settings.pluginPath);
     logger.log('Plugin installation complete');
   })
-  .catch(function (e) {
-    logger.error(`Plugin installation was unsuccessful due to error "${e.message}"`);
+  .catch((err) => {
+    logger.error(`Plugin installation was unsuccessful due to error "${err.message}"`);
     cleaner.cleanError();
     process.exit(70); // eslint-disable-line no-process-exit
   });

src/cli/plugin/pluginLogger.js

@@ -33,7 +33,7 @@ module.exports = function (settings) {
       data.pipe(process.stderr);
       return;
     }
-    process.stderr.write(data + '\n');
+    process.stderr.write(`${data}\n`);
     previousLineEnded = true;
   }