From 01678938fd47957c681bad5b268bc4b062718962 Mon Sep 17 00:00:00 2001 From: Wes Roberts Date: Wed, 23 Dec 2020 13:55:19 -0500 Subject: [PATCH 1/7] Restores local prebuilds, previously removed in #81 / a06925378d38ca821bfa93aa4c1fdedc253b2420 --- download.js | 92 +++++++++++++++++++++++++++++++---------------------- util.js | 5 +++ 2 files changed, 59 insertions(+), 38 deletions(-) diff --git a/download.js b/download.js index 13cb9f4..fdc0823 100644 --- a/download.js +++ b/download.js @@ -15,58 +15,74 @@ var mkdirp = require('mkdirp-classic') function downloadPrebuild (downloadUrl, opts, cb) { var cachedPrebuild = util.cachedPrebuild(downloadUrl) + var localPrebuild = util.localPrebuild(downloadUrl) var tempFile = util.tempFile(cachedPrebuild) var log = opts.log || noop - ensureNpmCacheDir(function (err) { - if (err) return onerror(err) + if (opts.nolocal) return download() - log.info('looking for cached prebuild @', cachedPrebuild) - fs.access(cachedPrebuild, fs.R_OK | fs.W_OK, function (err) { - if (!(err && err.code === 'ENOENT')) { - log.info('found cached prebuild') - return unpack() - } + log.info('looking for local prebuild @', localPrebuild) + fs.access(localPrebuild, fs.R_OK | fs.W_OK, function (err) { + if (err && err.code === 'ENOENT') { + return download() + } - log.http('request', 'GET ' + downloadUrl) - var reqOpts = proxy({ url: downloadUrl }, opts) + log.info('found local prebuild') + cachedPrebuild = localPrebuild + unpack() + }) - if (opts.token) { - reqOpts.url += '?access_token=' + opts.token - reqOpts.headers = { - 'User-Agent': 'simple-get', - Accept: 'application/octet-stream' + function download () { + ensureNpmCacheDir(function (err) { + if (err) return onerror(err) + + log.info('looking for cached prebuild @', cachedPrebuild) + fs.access(cachedPrebuild, fs.R_OK | fs.W_OK, function (err) { + if (!(err && err.code === 'ENOENT')) { + log.info('found cached prebuild') + return unpack() } - } - var req = get(reqOpts, function (err, res) { - if (err) return onerror(err) - log.http(res.statusCode, downloadUrl) - if (res.statusCode !== 200) return onerror() - mkdirp(util.prebuildCache(), function () { - log.info('downloading to @', tempFile) - pump(res, fs.createWriteStream(tempFile), function (err) { - if (err) return onerror(err) - fs.rename(tempFile, cachedPrebuild, function (err) { - if (err) return cb(err) - log.info('renaming to @', cachedPrebuild) - unpack() + log.http('request', 'GET ' + downloadUrl) + var reqOpts = proxy({ url: downloadUrl }, opts) + + if (opts.token) { + reqOpts.url += '?access_token=' + opts.token + reqOpts.headers = { + 'User-Agent': 'simple-get', + Accept: 'application/octet-stream' + } + } + + var req = get(reqOpts, function (err, res) { + if (err) return onerror(err) + log.http(res.statusCode, downloadUrl) + if (res.statusCode !== 200) return onerror() + mkdirp(util.prebuildCache(), function () { + log.info('downloading to @', tempFile) + pump(res, fs.createWriteStream(tempFile), function (err) { + if (err) return onerror(err) + fs.rename(tempFile, cachedPrebuild, function (err) { + if (err) return cb(err) + log.info('renaming to @', cachedPrebuild) + unpack() + }) }) }) }) - }) - req.setTimeout(30 * 1000, function () { - req.abort() + req.setTimeout(30 * 1000, function () { + req.abort() + }) }) - }) - function onerror (err) { - fs.unlink(tempFile, function () { - cb(err || error.noPrebuilts(opts)) - }) - } - }) + function onerror (err) { + fs.unlink(tempFile, function () { + cb(err || error.noPrebuilts(opts)) + }) + } + }) + } function unpack () { var binaryName diff --git a/util.js b/util.js index db358c9..0e6b95c 100644 --- a/util.js +++ b/util.js @@ -105,11 +105,16 @@ function packageOrigin (env, pkg) { } } +function localPrebuild (url) { + return path.join('prebuilds', path.basename(url)) +} + exports.getDownloadUrl = getDownloadUrl exports.getApiUrl = getApiUrl exports.getAssetUrl = getAssetUrl exports.urlTemplate = urlTemplate exports.cachedPrebuild = cachedPrebuild +exports.localPrebuild = localPrebuild exports.prebuildCache = prebuildCache exports.npmCache = npmCache exports.tempFile = tempFile From e434990ac1fbb09cf2ece5dc341beae68a8869bc Mon Sep 17 00:00:00 2001 From: Wes Roberts Date: Wed, 23 Dec 2020 17:00:19 -0500 Subject: [PATCH 2/7] Local prebuilds - adds env var, tests, and docs --- README.md | 14 ++++++++++++++ download.js | 2 +- rc.js | 1 + test/download-test.js | 38 +++++++++++++++++++++++++++++++++++++- test/util-test.js | 28 +++++++++++++++++++++++++++- util.js | 6 ++++-- 6 files changed, 84 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 06f2b7c..7946ccf 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,20 @@ So if you are installing `leveldown@1.2.3` the resulting url will be: http://overriden-host.com/overriden-path/v1.2.3/leveldown-v1.2.3-node-v57-win32-x64.tar.gz ``` +#### Local prebuilds + +If you want to use prebuilds from your local filesystem, you can use the `% your package name %_local_prebuilds` .npmrc variable to set a path to the folder containing prebuilds. For example: + +``` +leveldown_local_prebuilds=/path/to/prebuilds +``` + +This option will look directly in that folder for bundles created with `prebuild`, for example: + +``` +/path/to/prebuilds/leveldown-v1.2.3-node-v57-win32-x64.tar.gz +``` + ### Cache All prebuilt binaries are cached to minimize traffic. So first `prebuild-install` picks binaries from the cache and if no binary could be found, it will be downloaded. Depending on the environment, the cache folder is determined in the following order: diff --git a/download.js b/download.js index fdc0823..6565767 100644 --- a/download.js +++ b/download.js @@ -15,7 +15,7 @@ var mkdirp = require('mkdirp-classic') function downloadPrebuild (downloadUrl, opts, cb) { var cachedPrebuild = util.cachedPrebuild(downloadUrl) - var localPrebuild = util.localPrebuild(downloadUrl) + var localPrebuild = util.localPrebuild(downloadUrl, opts) var tempFile = util.tempFile(cachedPrebuild) var log = opts.log || noop diff --git a/rc.js b/rc.js index 41c4a94..3316cab 100644 --- a/rc.js +++ b/rc.js @@ -25,6 +25,7 @@ module.exports = function (pkg) { proxy: env.npm_config_proxy || env['http_proxy'] || env['HTTP_PROXY'], 'https-proxy': env.npm_config_https_proxy || env['https_proxy'] || env['HTTPS_PROXY'], 'local-address': env.npm_config_local_address, + 'local-prebuilds': 'prebuilds', 'tag-prefix': 'v', download: env.npm_config_download }, minimist(process.argv, { diff --git a/test/download-test.js b/test/download-test.js index 3ee4fac..e374aa8 100644 --- a/test/download-test.js +++ b/test/download-test.js @@ -89,6 +89,41 @@ test('cached prebuild', function (t) { }) }) +test('local prebuild', function (t) { + t.plan(6) + rm.sync(build) + + var opts = getOpts() + var downloadUrl = util.getDownloadUrl(opts) + var cachedPrebuild = util.cachedPrebuild(downloadUrl) + var localPrebuild = util.localPrebuild(downloadUrl, opts) + + t.ok(fs.existsSync(cachedPrebuild), 'cached prebuild exists') + fs.copyFileSync(cachedPrebuild, localPrebuild) + + var _createWriteStream = fs.createWriteStream + fs.createWriteStream = function (path) { + t.ok(/\.node$/i.test(path), 'this is the unpacked file') + return _createWriteStream(path) + } + + var _createReadStream = fs.createReadStream + fs.createReadStream = function (path) { + t.equal(path, localPrebuild, 'createReadStream called for localPrebuild') + return _createReadStream(path) + } + + t.equal(fs.existsSync(build), false, 'no build folder') + + download(downloadUrl, opts, function (err) { + t.error(err, 'no error') + t.equal(fs.existsSync(unpacked), true, unpacked + ' should exist') + fs.createReadStream = _createReadStream + fs.createWriteStream = _createWriteStream + rm.sync(localPrebuild) + }) +}) + test('non existing host should fail with no dangling temp file', function (t) { t.plan(3) @@ -225,6 +260,7 @@ function getOpts () { platform: process.platform, arch: process.arch, path: __dirname, - 'tag-prefix': 'v' + 'tag-prefix': 'v', + 'local-prebuilds': __dirname } } diff --git a/test/util-test.js b/test/util-test.js index 71252bd..08bdbb7 100644 --- a/test/util-test.js +++ b/test/util-test.js @@ -5,7 +5,7 @@ var util = require('../util') var path = require('path') test('prebuildCache() for different environments', function (t) { - var NPMCACHE = process.env.npm_config_cache + var NPMCACHE = process.env.npm_config_cache || '' delete process.env.npm_config_cache var APPDATA = process.env.APPDATA = 'somepathhere' t.equal(util.prebuildCache(), path.join(APPDATA, '/npm-cache/_prebuilds'), 'APPDATA set') @@ -190,3 +190,29 @@ test('getDownloadUrl() expands template to correct values', function (t) { t.equal(url3, url2, 'scope does not matter for download url') t.end() }) + +test('localPrebuild', function (t) { + var envProp = 'npm_config_a_native_module_local_prebuilds' + var basename = 'a-native-module-v1.4.0-node-v14-linux-x64.tar.gz' + var url = 'https://github.com/a-native-module/a-native-module/releases/download/v1.4.0/' + basename + var o1 = { + pkg: { + name: 'a-native-module' + } + } + var path1 = util.localPrebuild(url, o1) + t.equal(path1, path.join('prebuilds', basename)) + var o2 = { + pkg: { + name: 'a-native-module' + }, + 'local-prebuilds': path.join('', 'path', 'to', 'prebuilds') + } + var path2 = util.localPrebuild(url, o2) + t.equal(path2, path.join(o2['local-prebuilds'], basename), 'opts overrides default') + var envPrefix = path.join('', 'overriden', 'path', 'to', 'prebuilds') + process.env[envProp] = envPrefix + var path3 = util.localPrebuild(url, o2) + t.equal(path3, path.join(envPrefix, basename), 'env overrides opts') + t.end() +}) diff --git a/util.js b/util.js index 0e6b95c..68bd609 100644 --- a/util.js +++ b/util.js @@ -105,8 +105,10 @@ function packageOrigin (env, pkg) { } } -function localPrebuild (url) { - return path.join('prebuilds', path.basename(url)) +function localPrebuild (url, opts) { + var propName = 'npm_config_' + (opts.pkg.name || '').replace(/[^a-zA-Z0-9]/g, '_') + '_local_prebuilds' + var prefix = process.env[propName] || opts['local-prebuilds'] || 'prebuilds' + return path.join(prefix, path.basename(url)) } exports.getDownloadUrl = getDownloadUrl From 3710516b8684c615951d783d15dd979a8ddb5c78 Mon Sep 17 00:00:00 2001 From: Wes Roberts Date: Wed, 23 Dec 2020 17:30:47 -0500 Subject: [PATCH 3/7] Local prebuilds - fixes errors on Node 6 --- README.md | 2 ++ test/download-test.js | 5 ++++- test/util-test.js | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7946ccf..6ae67ff 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,8 @@ So if you are installing `leveldown@1.2.3` the resulting url will be: http://overriden-host.com/overriden-path/v1.2.3/leveldown-v1.2.3-node-v57-win32-x64.tar.gz ``` +Non-absolute paths resolve relative to the directory of the package invoking prebuild-install, e.g. for nested dependencies. + #### Local prebuilds If you want to use prebuilds from your local filesystem, you can use the `% your package name %_local_prebuilds` .npmrc variable to set a path to the folder containing prebuilds. For example: diff --git a/test/download-test.js b/test/download-test.js index e374aa8..efa14b5 100644 --- a/test/download-test.js +++ b/test/download-test.js @@ -99,7 +99,10 @@ test('local prebuild', function (t) { var localPrebuild = util.localPrebuild(downloadUrl, opts) t.ok(fs.existsSync(cachedPrebuild), 'cached prebuild exists') - fs.copyFileSync(cachedPrebuild, localPrebuild) + + // fs.copyFileSync() not available before Node 8.5 + var data = fs.readFileSync(cachedPrebuild); + fs.writeFileSync(localPrebuild, data); var _createWriteStream = fs.createWriteStream fs.createWriteStream = function (path) { diff --git a/test/util-test.js b/test/util-test.js index 08bdbb7..13c6d50 100644 --- a/test/util-test.js +++ b/test/util-test.js @@ -5,7 +5,7 @@ var util = require('../util') var path = require('path') test('prebuildCache() for different environments', function (t) { - var NPMCACHE = process.env.npm_config_cache || '' + var NPMCACHE = process.env.npm_config_cache delete process.env.npm_config_cache var APPDATA = process.env.APPDATA = 'somepathhere' t.equal(util.prebuildCache(), path.join(APPDATA, '/npm-cache/_prebuilds'), 'APPDATA set') From a8c1b5ef9ebcb5b70ebcab4b163dd491544b6baa Mon Sep 17 00:00:00 2001 From: Wes Roberts Date: Wed, 23 Dec 2020 17:32:50 -0500 Subject: [PATCH 4/7] Local prebuilds - fixes README change --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6ae67ff..d95d022 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,6 @@ So if you are installing `leveldown@1.2.3` the resulting url will be: http://overriden-host.com/overriden-path/v1.2.3/leveldown-v1.2.3-node-v57-win32-x64.tar.gz ``` -Non-absolute paths resolve relative to the directory of the package invoking prebuild-install, e.g. for nested dependencies. #### Local prebuilds @@ -113,6 +112,8 @@ This option will look directly in that folder for bundles created with `prebuild /path/to/prebuilds/leveldown-v1.2.3-node-v57-win32-x64.tar.gz ``` +Non-absolute paths resolve relative to the directory of the package invoking prebuild-install, e.g. for nested dependencies. + ### Cache All prebuilt binaries are cached to minimize traffic. So first `prebuild-install` picks binaries from the cache and if no binary could be found, it will be downloaded. Depending on the environment, the cache folder is determined in the following order: From e7c2f76976abfc1ccef515a574b39fce24a1bba7 Mon Sep 17 00:00:00 2001 From: Wes Roberts Date: Wed, 23 Dec 2020 17:38:27 -0500 Subject: [PATCH 5/7] Local prebuilds - fix formatting --- test/download-test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/download-test.js b/test/download-test.js index efa14b5..9228534 100644 --- a/test/download-test.js +++ b/test/download-test.js @@ -101,8 +101,8 @@ test('local prebuild', function (t) { t.ok(fs.existsSync(cachedPrebuild), 'cached prebuild exists') // fs.copyFileSync() not available before Node 8.5 - var data = fs.readFileSync(cachedPrebuild); - fs.writeFileSync(localPrebuild, data); + var data = fs.readFileSync(cachedPrebuild) + fs.writeFileSync(localPrebuild, data) var _createWriteStream = fs.createWriteStream fs.createWriteStream = function (path) { From 08b46c9f9a5ddbc1e1897209f73500de7ecb857e Mon Sep 17 00:00:00 2001 From: Wes Roberts Date: Wed, 23 Dec 2020 19:23:48 -0500 Subject: [PATCH 6/7] Local prebuilds - stylistic change to retrigger anomalous AppVeyor tests --- test/download-test.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/download-test.js b/test/download-test.js index 9228534..f8c5b39 100644 --- a/test/download-test.js +++ b/test/download-test.js @@ -101,8 +101,7 @@ test('local prebuild', function (t) { t.ok(fs.existsSync(cachedPrebuild), 'cached prebuild exists') // fs.copyFileSync() not available before Node 8.5 - var data = fs.readFileSync(cachedPrebuild) - fs.writeFileSync(localPrebuild, data) + fs.writeFileSync(localPrebuild, fs.readFileSync(cachedPrebuild)) var _createWriteStream = fs.createWriteStream fs.createWriteStream = function (path) { From afdddb80862292e6727495085d6583d9ee7b0c8a Mon Sep 17 00:00:00 2001 From: Wes Roberts Date: Fri, 29 Jan 2021 12:05:00 -0500 Subject: [PATCH 7/7] Local prebuilds - adds getEnvPrefix() to cinch duplicate code --- util.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/util.js b/util.js index 68bd609..eb392f6 100644 --- a/util.js +++ b/util.js @@ -60,8 +60,12 @@ function urlTemplate (opts) { return github(opts.pkg) + '/releases/download/{tag_prefix}{version}/' + packageName } +function getEnvPrefix (pkgName) { + return 'npm_config_' + (pkgName || '').replace(/[^a-zA-Z0-9]/g, '_') +} + function getHostMirrorUrl (opts) { - var propName = 'npm_config_' + (opts.pkg.name || '').replace(/[^a-zA-Z0-9]/g, '_') + '_binary_host' + var propName = getEnvPrefix(opts.pkg.name) + '_binary_host' return process.env[propName] || process.env[propName + '_mirror'] } @@ -106,7 +110,7 @@ function packageOrigin (env, pkg) { } function localPrebuild (url, opts) { - var propName = 'npm_config_' + (opts.pkg.name || '').replace(/[^a-zA-Z0-9]/g, '_') + '_local_prebuilds' + var propName = getEnvPrefix(opts.pkg.name) + '_local_prebuilds' var prefix = process.env[propName] || opts['local-prebuilds'] || 'prebuilds' return path.join(prefix, path.basename(url)) }