From 3ffc1c72ba9130591388c579846e6d369691ba7e Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Tue, 9 Aug 2016 18:50:19 +0200 Subject: [PATCH 1/4] add proxy support (with request module) - Use request module instead of Wreck - Add proxy support --- src/cli_plugin/install/download.js | 2 +- src/cli_plugin/install/downloaders/http.js | 47 ++++++++++++++-------- src/cli_plugin/install/index.js | 7 +++- src/cli_plugin/install/settings.js | 13 +++++- 4 files changed, 49 insertions(+), 20 deletions(-) diff --git a/src/cli_plugin/install/download.js b/src/cli_plugin/install/download.js index 871b170628fe8..8e67978c9bf96 100644 --- a/src/cli_plugin/install/download.js +++ b/src/cli_plugin/install/download.js @@ -10,7 +10,7 @@ export function _downloadSingle(settings, logger, sourceUrl) { if (/^file/.test(urlInfo.protocol)) { downloadPromise = downloadLocalFile(logger, decodeURI(urlInfo.path), settings.tempArchiveFile); } else if (/^https?/.test(urlInfo.protocol)) { - downloadPromise = downloadHttpFile(logger, sourceUrl, settings.tempArchiveFile, settings.timeout); + downloadPromise = downloadHttpFile(logger, sourceUrl, settings.tempArchiveFile, settings.timeout, settings.proxy); } else { downloadPromise = Promise.reject(new UnsupportedProtocolError()); } diff --git a/src/cli_plugin/install/downloaders/http.js b/src/cli_plugin/install/downloaders/http.js index 40069c4cd063e..4f57189df25f6 100644 --- a/src/cli_plugin/install/downloaders/http.js +++ b/src/cli_plugin/install/downloaders/http.js @@ -1,44 +1,46 @@ -import Wreck from 'wreck'; import Progress from '../progress'; import { fromNode as fn } from 'bluebird'; +import Request from 'request'; import { createWriteStream, unlinkSync } from 'fs'; -function sendRequest({ sourceUrl, timeout }) { - const maxRedirects = 11; //Because this one goes to 11. +function sendRequest({ sourceUrl, reqOptions }) { return fn(cb => { - const req = Wreck.request('GET', sourceUrl, { timeout, redirects: maxRedirects }, (err, resp) => { + const req = Request.head(sourceUrl, reqOptions, (err, resp) => { if (err) { if (err.code === 'ECONNREFUSED') { err = new Error('ENOTFOUND'); } - return cb(err); } if (resp.statusCode >= 400) { return cb(new Error('ENOTFOUND')); } - cb(null, { req, resp }); }); }); } -function downloadResponse({ resp, targetPath, progress }) { +function downloadResponse({ sourceUrl, reqOptions, targetPath, progress }) { return new Promise((resolve, reject) => { const writeStream = createWriteStream(targetPath); // if either stream errors, fail quickly - resp.on('error', reject); writeStream.on('error', reject); - // report progress as we download - resp.on('data', (chunk) => { - progress.progress(chunk.length); - }); + Request + // Get the plugin + .get(sourceUrl, reqOptions) + + // if either stream errors, fail quickly + .on('error', reject) - // write the download to the file system - resp.pipe(writeStream); + // report progress as we download + .on('data', (chunk) => { + progress.progress(chunk.length); + }) + // write the download to the file system + .pipe(writeStream); // when the write is done, we are done writeStream.on('finish', resolve); @@ -48,16 +50,27 @@ function downloadResponse({ resp, targetPath, progress }) { /* Responsible for managing http transfers */ -export default async function downloadUrl(logger, sourceUrl, targetPath, timeout) { +export default async function downloadUrl(logger, sourceUrl, targetPath, timeout, proxy) { + let reqOptions = { + 'timeout': timeout, + 'maxRedirects': 11, //Because this one goes to 11. + 'encoding': null + }; + + if (proxy) { + reqOptions.proxy = proxy; + logger.log(`Attempting to use the following proxy: ${proxy}`); + } + try { - const { req, resp } = await sendRequest({ sourceUrl, timeout }); + const { req, resp } = await sendRequest({ sourceUrl, reqOptions }); try { let totalSize = parseFloat(resp.headers['content-length']) || 0; const progress = new Progress(logger); progress.init(totalSize); - await downloadResponse({ resp, targetPath, progress }); + await downloadResponse({ sourceUrl, reqOptions, targetPath, progress }); progress.complete(); } catch (err) { diff --git a/src/cli_plugin/install/index.js b/src/cli_plugin/install/index.js index 78547d1db006b..6a956c576f52c 100644 --- a/src/cli_plugin/install/index.js +++ b/src/cli_plugin/install/index.js @@ -4,7 +4,7 @@ import install from './install'; import Logger from '../lib/logger'; import pkg from '../../utils/package_json'; import { getConfig } from '../../server/path'; -import { parse, parseMilliseconds } from './settings'; +import { parse, parseMilliseconds, parseProxy } from './settings'; import { find } from 'lodash'; function processCommand(command, options) { @@ -41,6 +41,11 @@ export default function pluginInstall(program) { 'path to the directory where plugins are stored', fromRoot('plugins') ) + .option( + '--proxy ', + 'proxy url to use', + parseProxy + ) .description('install a plugin', `Common examples: install x-pack diff --git a/src/cli_plugin/install/settings.js b/src/cli_plugin/install/settings.js index fd87342d5dae3..00ff7fc88c618 100644 --- a/src/cli_plugin/install/settings.js +++ b/src/cli_plugin/install/settings.js @@ -23,6 +23,16 @@ export function parseMilliseconds(val) { return result; }; +export function parseProxy(val) { + let result = val.trim(); + let regexp = /(http|https):\/\/(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?/; + if (!regexp.test(result)) { + throw new Error(`Invalid proxy name ${result}`); + } + + return result; +} + export function parse(command, options, kbnPackage) { const settings = { timeout: options.timeout || 0, @@ -31,7 +41,8 @@ export function parse(command, options, kbnPackage) { config: options.config || '', plugin: command, version: kbnPackage.version, - pluginDir: options.pluginDir || '' + pluginDir: options.pluginDir || '', + proxy: options.proxy || '' }; settings.urls = generateUrls(settings); From ef332eabb71209a9ac8b25a766a6b8dc0449c1b7 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Fri, 12 Aug 2016 17:35:50 +0200 Subject: [PATCH 2/4] don't send a HTTP HEAD request, but try to download file instead --- src/cli_plugin/install/downloaders/http.js | 45 ++++++++++------------ 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/src/cli_plugin/install/downloaders/http.js b/src/cli_plugin/install/downloaders/http.js index 4f57189df25f6..ae24046641f61 100644 --- a/src/cli_plugin/install/downloaders/http.js +++ b/src/cli_plugin/install/downloaders/http.js @@ -4,43 +4,40 @@ import Request from 'request'; import { createWriteStream, unlinkSync } from 'fs'; function sendRequest({ sourceUrl, reqOptions }) { - return fn(cb => { - const req = Request.head(sourceUrl, reqOptions, (err, resp) => { - if (err) { - if (err.code === 'ECONNREFUSED') { - err = new Error('ENOTFOUND'); - } - return cb(err); - } + return new Promise((resolve, reject) => { + const req = Request.get(sourceUrl, reqOptions); + req.on('response', (resp) => { if (resp.statusCode >= 400) { - return cb(new Error('ENOTFOUND')); + return reject(new Error('ENOTFOUND')); } - cb(null, { req, resp }); + return resolve({ req, resp }); + }); + + req.on('error', (err) => { + if (err.code === 'ECONNREFUSED') { + err = new Error('ENOTFOUND'); + } + return reject(err); }); }); } -function downloadResponse({ sourceUrl, reqOptions, targetPath, progress }) { +function downloadResponse({ resp, targetPath, progress }) { return new Promise((resolve, reject) => { const writeStream = createWriteStream(targetPath); // if either stream errors, fail quickly + resp.on('error', reject); writeStream.on('error', reject); - Request - // Get the plugin - .get(sourceUrl, reqOptions) - - // if either stream errors, fail quickly - .on('error', reject) + // report progress as we download + resp.on('data', (chunk) => { + progress.progress(chunk.length); + }); - // report progress as we download - .on('data', (chunk) => { - progress.progress(chunk.length); - }) - // write the download to the file system - .pipe(writeStream); + // write the download to the file system + resp.pipe(writeStream); // when the write is done, we are done writeStream.on('finish', resolve); @@ -70,7 +67,7 @@ export default async function downloadUrl(logger, sourceUrl, targetPath, timeout const progress = new Progress(logger); progress.init(totalSize); - await downloadResponse({ sourceUrl, reqOptions, targetPath, progress }); + await downloadResponse({ resp, targetPath, progress }); progress.complete(); } catch (err) { From 55bbb441db3cf34e44ea90e9662eb148e3621380 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Fri, 2 Sep 2016 16:21:52 +0200 Subject: [PATCH 3/4] fix some coding style --- src/cli_plugin/install/downloaders/http.js | 2 +- src/cli_plugin/install/settings.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cli_plugin/install/downloaders/http.js b/src/cli_plugin/install/downloaders/http.js index ae24046641f61..8b5e413c00006 100644 --- a/src/cli_plugin/install/downloaders/http.js +++ b/src/cli_plugin/install/downloaders/http.js @@ -48,7 +48,7 @@ function downloadResponse({ resp, targetPath, progress }) { Responsible for managing http transfers */ export default async function downloadUrl(logger, sourceUrl, targetPath, timeout, proxy) { - let reqOptions = { + const reqOptions = { 'timeout': timeout, 'maxRedirects': 11, //Because this one goes to 11. 'encoding': null diff --git a/src/cli_plugin/install/settings.js b/src/cli_plugin/install/settings.js index 00ff7fc88c618..b02e0e4381362 100644 --- a/src/cli_plugin/install/settings.js +++ b/src/cli_plugin/install/settings.js @@ -24,8 +24,8 @@ export function parseMilliseconds(val) { }; export function parseProxy(val) { - let result = val.trim(); - let regexp = /(http|https):\/\/(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?/; + const result = val.trim(); + const regexp = /(http|https):\/\/(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?/; if (!regexp.test(result)) { throw new Error(`Invalid proxy name ${result}`); } From 5a6d6805504e9b46a517575bb751e72983fb1616 Mon Sep 17 00:00:00 2001 From: "Thierno IB. BARRY" Date: Mon, 19 Sep 2016 16:32:52 +0200 Subject: [PATCH 4/4] remove wreck dependences --- package.json | 3 +-- tasks/build/download_node_builds.js | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 4a10c69b207cd..7a9f9978ba10b 100644 --- a/package.json +++ b/package.json @@ -156,8 +156,7 @@ "validate-npm-package-name": "2.2.2", "webpack": "1.12.15", "webpack-directory-name-as-main": "1.0.0", - "whatwg-fetch": "0.9.0", - "wreck": "6.2.0" + "whatwg-fetch": "0.9.0" }, "devDependencies": { "@elastic/eslint-config-kibana": "0.0.3", diff --git a/tasks/build/download_node_builds.js b/tasks/build/download_node_builds.js index cb74b55cec446..d05c488962fea 100644 --- a/tasks/build/download_node_builds.js +++ b/tasks/build/download_node_builds.js @@ -6,7 +6,7 @@ module.exports = function (grunt) { let { createGunzip } = require('zlib'); let { Extract } = require('tar'); let { rename } = require('fs'); - let wreck = require('wreck'); + let request = require('request'); let platforms = grunt.config.get('platforms'); let activeDownloads = []; @@ -21,7 +21,7 @@ module.exports = function (grunt) { } let resp = await fromNode(cb => { - let req = wreck.request('GET', platform.nodeUrl, null, function (err, resp) { + let req = request.get(platform.nodeUrl, function (err, resp) { if (err) { return cb(err); }