diff --git a/LICENSE b/LICENSE index 6de584a..778edb2 100644 --- a/LICENSE +++ b/LICENSE @@ -1,20 +1,48 @@ -Copyright Joyent, Inc. and other Node contributors. - -Permission is hereby granted, free of charge, to any person obtaining a -copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to permit -persons to whom the Software is furnished to do so, subject to the -following conditions: - -The above copyright notice and this permission notice shall be included -in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -USE OR OTHER DEALINGS IN THE SOFTWARE. +Node.js is licensed for use as follows: + +""" +Copyright Node.js contributors. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. +""" + +This license applies to parts of Node.js originating from the +https://github.com/joyent/node repository: + +""" +Copyright Joyent, Inc. and other Node contributors. All rights reserved. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. +""" + diff --git a/README.md b/README.md index 4d2aa00..13d827d 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,30 @@ -**string_decoder.js** (`require('string_decoder')`) from Node.js core +# string_decoder -Copyright Joyent, Inc. and other Node contributors. See LICENCE file for details. +***Node-core v7.0.0 string_decoder for userland*** -Version numbers match the versions found in Node core, e.g. 0.10.24 matches Node 0.10.24, likewise 0.11.10 matches Node 0.11.10. **Prefer the stable version over the unstable.** -The *build/* directory contains a build script that will scrape the source from the [joyent/node](https://github.com/joyent/node) repo given a specific Node version. \ No newline at end of file +[![NPM](https://nodei.co/npm/string_decoder.png?downloads=true&downloadRank=true)](https://nodei.co/npm/string_decoder/) +[![NPM](https://nodei.co/npm-dl/string_decoder.png?&months=6&height=3)](https://nodei.co/npm/string_decoder/) + + +[![Sauce Test Status](https://saucelabs.com/browser-matrix/string_decoder.svg)](https://saucelabs.com/u/string_decoder) + +```bash +npm install --save string_decoder +``` + +***Node-core string_decoderstring_decoder for userland*** + +This package is a mirror of the string_decoder implementation in Node-core. + +Full documentation may be found on the [Node.js website](https://nodejs.org/dist/v7.8.0/docs/api/). + +As of version 1.0.0 **string_decoder** uses semantic versioning. + +## Previous versions + +Previous version numbers match the versions found in Node core, e.g. 0.10.24 matches Node 0.10.24, likewise 0.11.10 matches Node 0.11.10. + +## Update + +The *build/* directory contains a build script that will scrape the source from the [nodejs/node](https://github.com/nodejs/node) repo given a specific Node version. diff --git a/build/build.js b/build/build.js index 46470cf..f52fd28 100755 --- a/build/build.js +++ b/build/build.js @@ -1,45 +1,89 @@ #!/usr/bin/env node -const hyperquest = require('hyperzip')(require('hyperdirect')) +const hyperquest = require('hyperquest') , bl = require('bl') , fs = require('fs') , path = require('path') - , cheerio = require('cheerio') - + , tar = require('tar-fs') + , gunzip = require('gunzip-maybe') + , babel = require('babel-core') + , glob = require('glob') + , pump = require('pump') + , rimraf = require('rimraf') + , encoding = 'utf8' + , urlRegex = /^https?:\/\// + , nodeVersion = process.argv[2] + , nodeVersionRegexString = '\\d+\\.\\d+\\.\\d+' + , usageVersionRegex = RegExp('^' + nodeVersionRegexString + '$') + , readmeVersionRegex = + RegExp('((?:Node-core )|(?:https\:\/\/nodejs\.org\/dist\/)v)' + nodeVersionRegexString, 'g') + + , readmePath = path.join(__dirname, '..', 'README.md') , files = require('./files') , testReplace = require('./test-replacements') - , srcurlpfx = 'https://raw.github.com/joyent/node/v' + process.argv[2] + '-release/' - , libsrcurl = srcurlpfx + 'lib/' - , testsrcurl = srcurlpfx + 'test/simple/' - , testlisturl = 'https://github.com/joyent/node/tree/v' + process.argv[2] + '-release/test/simple' - , libourroot = path.join(__dirname, '../') - , testourroot = path.join(__dirname, '../test/simple/') + , downloadurl = `https://nodejs.org/dist/v${nodeVersion}/node-v${nodeVersion}.tar.gz` + , src = path.join(__dirname, `node-v${nodeVersion}`) + , libsrcurl = path.join(src, 'lib/') + , testsrcurl = path.join(src, 'test/parallel/') + , libourroot = path.join(__dirname, '../lib/') + , testourroot = path.join(__dirname, '../test/parallel/') -function processFile (url, out, replacements) { - hyperquest(url).pipe(bl(function (err, data) { - if (err) - throw err +if (!usageVersionRegex.test(nodeVersion)) { + console.error('Usage: build.js xx.yy.zz') + return process.exit(1); +} + +// `inputLoc`: URL or local path. +function processFile (inputLoc, out, replacements) { + var file = fs.createReadStream(inputLoc, encoding) + file.pipe(bl(function (err, data) { + if (err) throw err + + console.log('Processing', inputLoc) data = data.toString() replacements.forEach(function (replacement) { - data = data.replace.apply(data, replacement) + const regexp = replacement[0] + var arg2 = replacement[1] + if (typeof arg2 === 'function') + arg2 = arg2.bind(data) + data = data.replace(regexp, arg2) }) - - fs.writeFile(out, data, 'utf8', function (err) { - if (err) - throw err + if (inputLoc.slice(-3) === '.js') { + const transformed = babel.transform(data, { + plugins: [ + 'transform-es2015-parameters', + 'transform-es2015-arrow-functions', + 'transform-es2015-block-scoping', + 'transform-es2015-template-literals', + 'transform-es2015-shorthand-properties', + 'transform-es2015-for-of', + 'transform-es2015-destructuring' + ] + }) + data = transformed.code + } + fs.writeFile(out, data, encoding, function (err) { + if (err) throw err console.log('Wrote', out) }) })) } - +function deleteOldTests(){ + const files = fs.readdirSync(path.join(__dirname, '..', 'test', 'parallel')); + for (let file of files) { + let name = path.join(__dirname, '..', 'test', 'parallel', file); + console.log('Removing', name); + fs.unlinkSync(name); + } +} function processLibFile (file) { var replacements = files[file] , url = libsrcurl + file - , out = path.join(libourroot, replacements.out || file) + , out = path.join(libourroot, file) processFile(url, out, replacements) } @@ -56,39 +100,68 @@ function processTestFile (file) { processFile(url, out, replacements) } +//-------------------------------------------------------------------- +// Download the release from nodejs.org +console.log(`Downloading ${downloadurl}`) +pump( + hyperquest(downloadurl), + gunzip(), + tar.extract(__dirname), + function (err) { + if (err) { + throw err + } -if (!/0\.1\d\.\d+/.test(process.argv[2])) { - console.log('Usage: build.js ') - return process.exit(-1) -} + //-------------------------------------------------------------------- + // Grab & process files in ../lib/ -//-------------------------------------------------------------------- -// Grab & process files in ../lib/ + Object.keys(files).forEach(processLibFile) -Object.keys(files).forEach(processLibFile) -//-------------------------------------------------------------------- -// Discover, grab and process all test-string-decoder* files on joyent/node + //-------------------------------------------------------------------- + // Discover, grab and process all test-stream* files on the given release + + glob(path.join(testsrcurl, 'test-string-decoder*.js'), function (err, list) { + if (err) { + throw err + } -hyperquest(testlisturl).pipe(bl(function (err, data) { - if (err) - throw err + list.forEach(function (file) { + file = path.basename(file) + processTestFile(file) + }) + }) - var $ = cheerio.load(data.toString()) - $('table.files .js-directory-link').each(function () { - var file = $(this).text() - if (/^test-string-decoder/.test(file) || file == 'common.js') - processTestFile(file) - }) -})) + //-------------------------------------------------------------------- + // Grab the nodejs/node test/common.js -//-------------------------------------------------------------------- -// Grab the joyent/node test/common.js + processFile( + testsrcurl.replace(/parallel\/$/, 'common.js') + , path.join(testourroot, '../common.js') + , testReplace['common.js'] + ) -processFile( - testsrcurl + '../common.js' - , path.join(testourroot, '../common.js') - , testReplace['common.js'] -) \ No newline at end of file + //-------------------------------------------------------------------- + // Update Node version in README + + processFile(readmePath, readmePath, [ + [readmeVersionRegex, "$1" + nodeVersion] + ]) + } +) + +// delete the current contents of test/parallel so if node removes any tests +// they are removed here +deleteOldTests(); + +process.once('beforeExit', function () { + rimraf(src, function (err) { + if (err) { + throw err + } + + console.log('Removed', src) + }) +}) diff --git a/build/files.js b/build/files.js index 7396a4f..647cf1c 100644 --- a/build/files.js +++ b/build/files.js @@ -8,14 +8,31 @@ module.exports['string_decoder.js'] = [ - // pull in Bufer as a require - // add Buffer.isEncoding where missing + // we do not need internal/util anymore [ - /^(\/\/ USE OR OTHER DEALINGS IN THE SOFTWARE\.)/m - , '$1\n\nvar Buffer = require(\'buffer\').Buffer;' - + '\n' - + '\nvar isBufferEncoding = Buffer.isEncoding' + /const internalUtil = require\('internal\/util'\);/ + , '' + ] + + , [ + /const Buffer = require\('buffer'\).Buffer;/ + , 'var Buffer = require(\'buffer\').Buffer;\n' + + 'var bufferShim = require(\'buffer-shims\');' + ] + + // allocUnsafe + + , [ + /Buffer\.((?:alloc)|(?:allocUnsafe)|(?:from))/g, + 'bufferShim.$1' + ] + + // add Buffer.isEncoding where missing + , [ + /const isEncoding = Buffer\[internalUtil.kIsEncodingSymbol\];/ + , '\nvar isEncoding = Buffer.isEncoding' + '\n || function(encoding) {' + + '\n encoding = \'\' + encoding' + '\n switch (encoding && encoding.toLowerCase()) {' + '\n case \'hex\': case \'utf8\': case \'utf-8\': case \'ascii\': case \'binary\': case \'base64\': case \'ucs2\': case \'ucs-2\': case \'utf16le\': case \'utf-16le\': case \'raw\': return true;' + '\n default: return false;' @@ -23,14 +40,46 @@ module.exports['string_decoder.js'] = [ + '\n }' + '\n' + + '\nfunction _normalizeEncoding(enc) {' + + '\n if (!enc) return \'utf8\';' + + '\n var retried;' + + '\n while (true) {' + + '\n switch (enc) {' + + '\n case \'utf8\':' + + '\n case \'utf-8\':' + + '\n return \'utf8\';' + + '\n case \'ucs2\':' + + '\n case \'ucs-2\':' + + '\n case \'utf16le\':' + + '\n case \'utf-16le\':' + + '\n return \'utf16le\';' + + '\n case \'latin1\':' + + '\n case \'binary\':' + + '\n return \'latin1\';' + + '\n case \'base64\':' + + '\n case \'ascii\':' + + '\n case \'hex\':' + + '\n return enc;' + + '\n default:' + + '\n if (retried) return; // undefined' + + '\n enc = (\'\' + enc).toLowerCase();' + + '\n retried = true;' + + '\n }' + + '\n }' + + '\n };' ] + // use custom Buffer.isEncoding reference , [ /Buffer\.isEncoding\(/g - , 'isBufferEncoding\(' + , 'isEncoding\(' ] -] + // use _normalizeEncoding everywhere + , [ + /internalUtil\.normalizeEncoding/g + , '_normalizeEncoding' + ] -module.exports['string_decoder.js'].out = 'index.js' \ No newline at end of file +] diff --git a/build/package.json b/build/package.json index f7ee6a4..29dc1c2 100644 --- a/build/package.json +++ b/build/package.json @@ -4,9 +4,20 @@ "description": "", "main": "build.js", "dependencies": { - "bl": "~0.6.0", - "hyperzip": "0.0.0", - "hyperdirect": "0.0.0", - "cheerio": "~0.13.1" + "babel-core": "^6.5.2", + "babel-plugin-transform-es2015-arrow-functions": "^6.5.2", + "babel-plugin-transform-es2015-block-scoping": "^6.5.0", + "babel-plugin-transform-es2015-destructuring": "^6.18.0", + "babel-plugin-transform-es2015-for-of": "^6.8.0", + "babel-plugin-transform-es2015-parameters": "^6.11.4", + "babel-plugin-transform-es2015-shorthand-properties": "^6.8.0", + "babel-plugin-transform-es2015-template-literals": "^6.8.0", + "bl": "^1.2.0", + "glob": "^7.1.1", + "gunzip-maybe": "^1.4.0", + "hyperquest": "^2.1.2", + "pump": "^1.0.2", + "rimraf": "^2.6.1", + "tar-fs": "^1.15.1" } } diff --git a/build/test-replacements.js b/build/test-replacements.js index 5bbf602..8989250 100644 --- a/build/test-replacements.js +++ b/build/test-replacements.js @@ -4,6 +4,16 @@ module.exports.all = [ , 'require(\'../../\')' ] + , [ + /Buffer\.((?:alloc)|(?:allocUnsafe)|(?:from))/g, + 'bufferShim.$1' + ] + + , [ + /^('use strict';)$/m, + '$1\nconst bufferShim = require(\'buffer-shims\');' + ] + ] module.exports['common.js'] = [ @@ -21,4 +31,33 @@ module.exports['common.js'] = [ /^ global];$/m , ' global].filter(Boolean);' ] + + , [ + /^/ + , 'require(\'babel-polyfill\');' + ] + + , [ + /^( for \(var x in global\) \{|function leakedGlobals\(\) \{)$/m + , ' /**/\n' + + ' if (typeof constructor == \'function\')\n' + + ' knownGlobals.push(constructor);\n' + + ' if (typeof DTRACE_NET_SOCKET_READ == \'function\')\n' + + ' knownGlobals.push(DTRACE_NET_SOCKET_READ);\n' + + ' if (typeof DTRACE_NET_SOCKET_WRITE == \'function\')\n' + + ' knownGlobals.push(DTRACE_NET_SOCKET_WRITE);\n' + + ' if (global.__coverage__)\n' + + ' knownGlobals.push(__coverage__);\n' + + '\'core,__core-js_shared__,Promise,Map,Set,WeakMap,WeakSet,Reflect,System,asap,Observable,regeneratorRuntime,_babelPolyfill\'.split(\',\').filter(function (item) { return typeof global[item] !== undefined}).forEach(function (item) {knownGlobals.push(global[item])})' + + ' /**/\n\n$1' + ] +] + +module.exports['test-string-decoder.js'] = [ + // test removed because it is V8-version dependant. + [ + /test\('utf-8', bufferShim.from\('EDA0B5EDB08D'.*\n.*\n/ + , '' + ] ] + diff --git a/index.js b/index.js deleted file mode 100644 index 2e44a03..0000000 --- a/index.js +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright Joyent, Inc. and other Node contributors. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to permit -// persons to whom the Software is furnished to do so, subject to the -// following conditions: -// -// The above copyright notice and this permission notice shall be included -// in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -// USE OR OTHER DEALINGS IN THE SOFTWARE. - -var Buffer = require('buffer').Buffer; - -var isBufferEncoding = Buffer.isEncoding - || function(encoding) { - switch (encoding && encoding.toLowerCase()) { - case 'hex': case 'utf8': case 'utf-8': case 'ascii': case 'binary': case 'base64': case 'ucs2': case 'ucs-2': case 'utf16le': case 'utf-16le': case 'raw': return true; - default: return false; - } - } - - -function assertEncoding(encoding) { - if (encoding && !isBufferEncoding(encoding)) { - throw new Error('Unknown encoding: ' + encoding); - } -} - -var StringDecoder = exports.StringDecoder = function(encoding) { - this.encoding = (encoding || 'utf8').toLowerCase().replace(/[-_]/, ''); - assertEncoding(encoding); - switch (this.encoding) { - case 'utf8': - // CESU-8 represents each of Surrogate Pair by 3-bytes - this.surrogateSize = 3; - break; - case 'ucs2': - case 'utf16le': - // UTF-16 represents each of Surrogate Pair by 2-bytes - this.surrogateSize = 2; - this.detectIncompleteChar = utf16DetectIncompleteChar; - break; - case 'base64': - // Base-64 stores 3 bytes in 4 chars, and pads the remainder. - this.surrogateSize = 3; - this.detectIncompleteChar = base64DetectIncompleteChar; - break; - default: - this.write = passThroughWrite; - return; - } - - this.charBuffer = new Buffer(6); - this.charReceived = 0; - this.charLength = 0; -}; - - -StringDecoder.prototype.write = function(buffer) { - var charStr = ''; - var offset = 0; - - // if our last write ended with an incomplete multibyte character - while (this.charLength) { - // determine how many remaining bytes this buffer has to offer for this char - var i = (buffer.length >= this.charLength - this.charReceived) ? - this.charLength - this.charReceived : - buffer.length; - - // add the new bytes to the char buffer - buffer.copy(this.charBuffer, this.charReceived, offset, i); - this.charReceived += (i - offset); - offset = i; - - if (this.charReceived < this.charLength) { - // still not enough chars in this buffer? wait for more ... - return ''; - } - - // get the character that was split - charStr = this.charBuffer.slice(0, this.charLength).toString(this.encoding); - - // lead surrogate (D800-DBFF) is also the incomplete character - var charCode = charStr.charCodeAt(charStr.length - 1); - if (charCode >= 0xD800 && charCode <= 0xDBFF) { - this.charLength += this.surrogateSize; - charStr = ''; - continue; - } - this.charReceived = this.charLength = 0; - - // if there are no more bytes in this buffer, just emit our char - if (i == buffer.length) return charStr; - - // otherwise cut off the characters end from the beginning of this buffer - buffer = buffer.slice(i, buffer.length); - break; - } - - var lenIncomplete = this.detectIncompleteChar(buffer); - - var end = buffer.length; - if (this.charLength) { - // buffer the incomplete character bytes we got - buffer.copy(this.charBuffer, 0, buffer.length - lenIncomplete, end); - this.charReceived = lenIncomplete; - end -= lenIncomplete; - } - - charStr += buffer.toString(this.encoding, 0, end); - - var end = charStr.length - 1; - var charCode = charStr.charCodeAt(end); - // lead surrogate (D800-DBFF) is also the incomplete character - if (charCode >= 0xD800 && charCode <= 0xDBFF) { - var size = this.surrogateSize; - this.charLength += size; - this.charReceived += size; - this.charBuffer.copy(this.charBuffer, size, 0, size); - this.charBuffer.write(charStr.charAt(charStr.length - 1), this.encoding); - return charStr.substring(0, end); - } - - // or just emit the charStr - return charStr; -}; - -StringDecoder.prototype.detectIncompleteChar = function(buffer) { - // determine how many bytes we have to check at the end of this buffer - var i = (buffer.length >= 3) ? 3 : buffer.length; - - // Figure out if one of the last i bytes of our buffer announces an - // incomplete char. - for (; i > 0; i--) { - var c = buffer[buffer.length - i]; - - // See http://en.wikipedia.org/wiki/UTF-8#Description - - // 110XXXXX - if (i == 1 && c >> 5 == 0x06) { - this.charLength = 2; - break; - } - - // 1110XXXX - if (i <= 2 && c >> 4 == 0x0E) { - this.charLength = 3; - break; - } - - // 11110XXX - if (i <= 3 && c >> 3 == 0x1E) { - this.charLength = 4; - break; - } - } - - return i; -}; - -StringDecoder.prototype.end = function(buffer) { - var res = ''; - if (buffer && buffer.length) - res = this.write(buffer); - - if (this.charReceived) { - var cr = this.charReceived; - var buf = this.charBuffer; - var enc = this.encoding; - res += buf.slice(0, cr).toString(enc); - } - - return res; -}; - -function passThroughWrite(buffer) { - return buffer.toString(this.encoding); -} - -function utf16DetectIncompleteChar(buffer) { - var incomplete = this.charReceived = buffer.length % 2; - this.charLength = incomplete ? 2 : 0; - return incomplete; -} - -function base64DetectIncompleteChar(buffer) { - var incomplete = this.charReceived = buffer.length % 3; - this.charLength = incomplete ? 3 : 0; - return incomplete; -} diff --git a/lib/string_decoder.js b/lib/string_decoder.js new file mode 100644 index 0000000..696d7ab --- /dev/null +++ b/lib/string_decoder.js @@ -0,0 +1,273 @@ +'use strict'; + +var Buffer = require('buffer').Buffer; +var bufferShim = require('buffer-shims'); + +var isEncoding = Buffer.isEncoding || function (encoding) { + encoding = '' + encoding; + switch (encoding && encoding.toLowerCase()) { + case 'hex':case 'utf8':case 'utf-8':case 'ascii':case 'binary':case 'base64':case 'ucs2':case 'ucs-2':case 'utf16le':case 'utf-16le':case 'raw': + return true; + default: + return false; + } +}; + +function _normalizeEncoding(enc) { + if (!enc) return 'utf8'; + var retried; + while (true) { + switch (enc) { + case 'utf8': + case 'utf-8': + return 'utf8'; + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + return 'utf16le'; + case 'latin1': + case 'binary': + return 'latin1'; + case 'base64': + case 'ascii': + case 'hex': + return enc; + default: + if (retried) return; // undefined + enc = ('' + enc).toLowerCase(); + retried = true; + } + } +}; + +// Do not cache `Buffer.isEncoding` when checking encoding names as some +// modules monkey-patch it to support additional encodings +function normalizeEncoding(enc) { + var nenc = _normalizeEncoding(enc); + if (typeof nenc !== 'string' && (Buffer.isEncoding === isEncoding || !isEncoding(enc))) throw new Error('Unknown encoding: ' + enc); + return nenc || enc; +} + +// StringDecoder provides an interface for efficiently splitting a series of +// buffers into a series of JS strings without breaking apart multi-byte +// characters. +exports.StringDecoder = StringDecoder; +function StringDecoder(encoding) { + this.encoding = normalizeEncoding(encoding); + var nb; + switch (this.encoding) { + case 'utf16le': + this.text = utf16Text; + this.end = utf16End; + nb = 4; + break; + case 'utf8': + this.fillLast = utf8FillLast; + nb = 4; + break; + case 'base64': + this.text = base64Text; + this.end = base64End; + nb = 3; + break; + default: + this.write = simpleWrite; + this.end = simpleEnd; + return; + } + this.lastNeed = 0; + this.lastTotal = 0; + this.lastChar = bufferShim.allocUnsafe(nb); +} + +StringDecoder.prototype.write = function (buf) { + if (buf.length === 0) return ''; + var r; + var i; + if (this.lastNeed) { + r = this.fillLast(buf); + if (r === undefined) return ''; + i = this.lastNeed; + this.lastNeed = 0; + } else { + i = 0; + } + if (i < buf.length) return r ? r + this.text(buf, i) : this.text(buf, i); + return r || ''; +}; + +StringDecoder.prototype.end = utf8End; + +// Returns only complete characters in a Buffer +StringDecoder.prototype.text = utf8Text; + +// Attempts to complete a partial non-UTF-8 character using bytes from a Buffer +StringDecoder.prototype.fillLast = function (buf) { + if (this.lastNeed <= buf.length) { + buf.copy(this.lastChar, this.lastTotal - this.lastNeed, 0, this.lastNeed); + return this.lastChar.toString(this.encoding, 0, this.lastTotal); + } + buf.copy(this.lastChar, this.lastTotal - this.lastNeed, 0, buf.length); + this.lastNeed -= buf.length; +}; + +// Checks the type of a UTF-8 byte, whether it's ASCII, a leading byte, or a +// continuation byte. +function utf8CheckByte(byte) { + if (byte <= 0x7F) return 0;else if (byte >> 5 === 0x06) return 2;else if (byte >> 4 === 0x0E) return 3;else if (byte >> 3 === 0x1E) return 4; + return -1; +} + +// Checks at most 3 bytes at the end of a Buffer in order to detect an +// incomplete multi-byte UTF-8 character. The total number of bytes (2, 3, or 4) +// needed to complete the UTF-8 character (if applicable) are returned. +function utf8CheckIncomplete(self, buf, i) { + var j = buf.length - 1; + if (j < i) return 0; + var nb = utf8CheckByte(buf[j]); + if (nb >= 0) { + if (nb > 0) self.lastNeed = nb - 1; + return nb; + } + if (--j < i) return 0; + nb = utf8CheckByte(buf[j]); + if (nb >= 0) { + if (nb > 0) self.lastNeed = nb - 2; + return nb; + } + if (--j < i) return 0; + nb = utf8CheckByte(buf[j]); + if (nb >= 0) { + if (nb > 0) { + if (nb === 2) nb = 0;else self.lastNeed = nb - 3; + } + return nb; + } + return 0; +} + +// Validates as many continuation bytes for a multi-byte UTF-8 character as +// needed or are available. If we see a non-continuation byte where we expect +// one, we "replace" the validated continuation bytes we've seen so far with +// UTF-8 replacement characters ('\ufffd'), to match v8's UTF-8 decoding +// behavior. The continuation byte check is included three times in the case +// where all of the continuation bytes for a character exist in the same buffer. +// It is also done this way as a slight performance increase instead of using a +// loop. +function utf8CheckExtraBytes(self, buf, p) { + if ((buf[0] & 0xC0) !== 0x80) { + self.lastNeed = 0; + return '\ufffd'.repeat(p); + } + if (self.lastNeed > 1 && buf.length > 1) { + if ((buf[1] & 0xC0) !== 0x80) { + self.lastNeed = 1; + return '\ufffd'.repeat(p + 1); + } + if (self.lastNeed > 2 && buf.length > 2) { + if ((buf[2] & 0xC0) !== 0x80) { + self.lastNeed = 2; + return '\ufffd'.repeat(p + 2); + } + } + } +} + +// Attempts to complete a multi-byte UTF-8 character using bytes from a Buffer. +function utf8FillLast(buf) { + var p = this.lastTotal - this.lastNeed; + var r = utf8CheckExtraBytes(this, buf, p); + if (r !== undefined) return r; + if (this.lastNeed <= buf.length) { + buf.copy(this.lastChar, p, 0, this.lastNeed); + return this.lastChar.toString(this.encoding, 0, this.lastTotal); + } + buf.copy(this.lastChar, p, 0, buf.length); + this.lastNeed -= buf.length; +} + +// Returns all complete UTF-8 characters in a Buffer. If the Buffer ended on a +// partial character, the character's bytes are buffered until the required +// number of bytes are available. +function utf8Text(buf, i) { + var total = utf8CheckIncomplete(this, buf, i); + if (!this.lastNeed) return buf.toString('utf8', i); + this.lastTotal = total; + var end = buf.length - (total - this.lastNeed); + buf.copy(this.lastChar, 0, end); + return buf.toString('utf8', i, end); +} + +// For UTF-8, a replacement character for each buffered byte of a (partial) +// character needs to be added to the output. +function utf8End(buf) { + var r = buf && buf.length ? this.write(buf) : ''; + if (this.lastNeed) return r + '\ufffd'.repeat(this.lastTotal - this.lastNeed); + return r; +} + +// UTF-16LE typically needs two bytes per character, but even if we have an even +// number of bytes available, we need to check if we end on a leading/high +// surrogate. In that case, we need to wait for the next two bytes in order to +// decode the last character properly. +function utf16Text(buf, i) { + if ((buf.length - i) % 2 === 0) { + var r = buf.toString('utf16le', i); + if (r) { + var c = r.charCodeAt(r.length - 1); + if (c >= 0xD800 && c <= 0xDBFF) { + this.lastNeed = 2; + this.lastTotal = 4; + this.lastChar[0] = buf[buf.length - 2]; + this.lastChar[1] = buf[buf.length - 1]; + return r.slice(0, -1); + } + } + return r; + } + this.lastNeed = 1; + this.lastTotal = 2; + this.lastChar[0] = buf[buf.length - 1]; + return buf.toString('utf16le', i, buf.length - 1); +} + +// For UTF-16LE we do not explicitly append special replacement characters if we +// end on a partial character, we simply let v8 handle that. +function utf16End(buf) { + var r = buf && buf.length ? this.write(buf) : ''; + if (this.lastNeed) { + var end = this.lastTotal - this.lastNeed; + return r + this.lastChar.toString('utf16le', 0, end); + } + return r; +} + +function base64Text(buf, i) { + var n = (buf.length - i) % 3; + if (n === 0) return buf.toString('base64', i); + this.lastNeed = 3 - n; + this.lastTotal = 3; + if (n === 1) { + this.lastChar[0] = buf[buf.length - 1]; + } else { + this.lastChar[0] = buf[buf.length - 2]; + this.lastChar[1] = buf[buf.length - 1]; + } + return buf.toString('base64', i, buf.length - n); +} + +function base64End(buf) { + var r = buf && buf.length ? this.write(buf) : ''; + if (this.lastNeed) return r + this.lastChar.toString('base64', 0, 3 - this.lastNeed); + return r; +} + +// Pass bytes on through for single-byte encodings (e.g. ascii, latin1, hex) +function simpleWrite(buf) { + return buf.toString(this.encoding); +} + +function simpleEnd(buf) { + return buf && buf.length ? this.write(buf) : ''; +} \ No newline at end of file diff --git a/package.json b/package.json index 939a3cd..94668a8 100644 --- a/package.json +++ b/package.json @@ -2,13 +2,16 @@ "name": "string_decoder", "version": "0.11.10-3", "description": "The string_decoder module from Node core", - "main": "index.js", - "dependencies": {}, + "main": "lib/string_decoder.js", + "dependencies": { + "buffer-shims": "^1.0.0" + }, "devDependencies": { + "babel-polyfill": "^6.23.0", "tap": "~0.4.8" }, "scripts": { - "test": "tap test/simple/*.js" + "test": "tap test/parallel/*.js" }, "repository": { "type": "git", diff --git a/test/common.js b/test/common.js index 9af8964..e879f8e 100644 --- a/test/common.js +++ b/test/common.js @@ -1,193 +1,384 @@ -// Copyright Joyent, Inc. and other Node contributors. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to permit -// persons to whom the Software is furnished to do so, subject to the -// following conditions: -// -// The above copyright notice and this permission notice shall be included -// in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -// USE OR OTHER DEALINGS IN THE SOFTWARE. - +require('babel-polyfill'); /* eslint-disable required-modules */ +'use strict'; var path = require('path'); +var fs = require('fs'); var assert = require('assert'); +var os = require('os'); +var child_process = require('child_process'); +var stream = require('stream'); +var util = require('util'); +var Timer = process.binding('timer_wrap').Timer; +var execSync = require('child_process').execSync; + +var testRoot = process.env.NODE_TEST_DIR ? fs.realpathSync(process.env.NODE_TEST_DIR) : __dirname; -exports.testDir = path.dirname(__filename); -exports.fixturesDir = path.join(exports.testDir, 'fixtures'); -exports.libDir = path.join(exports.testDir, '../lib'); -exports.tmpDir = path.join(exports.testDir, 'tmp'); +exports.fixturesDir = path.join(__dirname, 'fixtures'); +exports.tmpDirName = 'tmp'; +// PORT should match the definition in test/testpy/__init__.py. exports.PORT = +process.env.NODE_COMMON_PORT || 12346; +exports.isWindows = process.platform === 'win32'; +exports.isWOW64 = exports.isWindows && process.env.PROCESSOR_ARCHITEW6432 !== undefined; +exports.isAix = process.platform === 'aix'; +exports.isLinuxPPCBE = process.platform === 'linux' && process.arch === 'ppc64' && os.endianness() === 'BE'; +exports.isSunOS = process.platform === 'sunos'; +exports.isFreeBSD = process.platform === 'freebsd'; +exports.isLinux = process.platform === 'linux'; +exports.isOSX = process.platform === 'darwin'; + +exports.enoughTestMem = os.totalmem() > 0x40000000; /* 1 Gb */ + +var cpus = os.cpus(); +exports.enoughTestCpu = Array.isArray(cpus) && (cpus.length > 1 || cpus[0].speed > 999); + +exports.rootDir = exports.isWindows ? 'c:\\' : '/'; +exports.buildType = process.config.target_defaults.default_configuration; + +function rimrafSync(p) { + var st = void 0; + try { + st = fs.lstatSync(p); + } catch (e) { + if (e.code === 'ENOENT') return; + } + + try { + if (st && st.isDirectory()) rmdirSync(p, null);else fs.unlinkSync(p); + } catch (e) { + if (e.code === 'ENOENT') return; + if (e.code === 'EPERM') return rmdirSync(p, e); + if (e.code !== 'EISDIR') throw e; + rmdirSync(p, e); + } +} + +function rmdirSync(p, originalEr) { + try { + fs.rmdirSync(p); + } catch (e) { + if (e.code === 'ENOTDIR') throw originalEr; + if (e.code === 'ENOTEMPTY' || e.code === 'EEXIST' || e.code === 'EPERM') { + var enc = exports.isLinux ? 'buffer' : 'utf8'; + fs.readdirSync(p, enc).forEach(function (f) { + if (f instanceof Buffer) { + var buf = Buffer.concat([Buffer.from(p), Buffer.from(path.sep), f]); + rimrafSync(buf); + } else { + rimrafSync(path.join(p, f)); + } + }); + fs.rmdirSync(p); + } + } +} + +exports.refreshTmpDir = function () { + rimrafSync(exports.tmpDir); + fs.mkdirSync(exports.tmpDir); +}; + +if (process.env.TEST_THREAD_ID) { + exports.PORT += process.env.TEST_THREAD_ID * 100; + exports.tmpDirName += '.' + process.env.TEST_THREAD_ID; +} +exports.tmpDir = path.join(testRoot, exports.tmpDirName); + +var opensslCli = null; +var inFreeBSDJail = null; +var localhostIPv4 = null; + +exports.localIPv6Hosts = ['localhost']; +if (exports.isLinux) { + exports.localIPv6Hosts = [ + // Debian/Ubuntu + 'ip6-localhost', 'ip6-loopback', + + // SUSE + 'ipv6-localhost', 'ipv6-loopback', + + // Typically universal + 'localhost']; +} + +Object.defineProperty(exports, 'inFreeBSDJail', { + get: function () { + if (inFreeBSDJail !== null) return inFreeBSDJail; + + if (exports.isFreeBSD && child_process.execSync('sysctl -n security.jail.jailed').toString() === '1\n') { + inFreeBSDJail = true; + } else { + inFreeBSDJail = false; + } + return inFreeBSDJail; + } +}); + +Object.defineProperty(exports, 'localhostIPv4', { + get: function () { + if (localhostIPv4 !== null) return localhostIPv4; + + if (exports.inFreeBSDJail) { + // Jailed network interfaces are a bit special - since we need to jump + // through loops, as well as this being an exception case, assume the + // user will provide this instead. + if (process.env.LOCALHOST) { + localhostIPv4 = process.env.LOCALHOST; + } else { + console.error('Looks like we\'re in a FreeBSD Jail. ' + 'Please provide your default interface address ' + 'as LOCALHOST or expect some tests to fail.'); + } + } + + if (localhostIPv4 === null) localhostIPv4 = '127.0.0.1'; + + return localhostIPv4; + } +}); + +// opensslCli defined lazily to reduce overhead of spawnSync +Object.defineProperty(exports, 'opensslCli', { get: function () { + if (opensslCli !== null) return opensslCli; + + if (process.config.variables.node_shared_openssl) { + // use external command + opensslCli = 'openssl'; + } else { + // use command built from sources included in Node.js repository + opensslCli = path.join(path.dirname(process.execPath), 'openssl-cli'); + } -if (process.platform === 'win32') { + if (exports.isWindows) opensslCli += '.exe'; + + var opensslCmd = child_process.spawnSync(opensslCli, ['version']); + if (opensslCmd.status !== 0 || opensslCmd.error !== undefined) { + // openssl command cannot be executed + opensslCli = false; + } + return opensslCli; + }, enumerable: true }); + +Object.defineProperty(exports, 'hasCrypto', { + get: function () { + return process.versions.openssl ? true : false; + } +}); + +Object.defineProperty(exports, 'hasFipsCrypto', { + get: function () { + return exports.hasCrypto && require('crypto').fips; + } +}); + +if (exports.isWindows) { exports.PIPE = '\\\\.\\pipe\\libuv-test'; - exports.opensslCli = path.join(process.execPath, '..', 'openssl-cli.exe'); + if (process.env.TEST_THREAD_ID) { + exports.PIPE += '.' + process.env.TEST_THREAD_ID; + } } else { exports.PIPE = exports.tmpDir + '/test.sock'; - exports.opensslCli = path.join(process.execPath, '..', 'openssl-cli'); } -var util = require('util'); -for (var i in util) exports[i] = util[i]; -//for (var i in exports) global[i] = exports[i]; - -function protoCtrChain(o) { - var result = []; - for (; o; o = o.__proto__) { result.push(o.constructor); } - return result.join(); -} +var ifaces = os.networkInterfaces(); +exports.hasIPv6 = Object.keys(ifaces).some(function (name) { + return (/lo/.test(name) && ifaces[name].some(function (info) { + return info.family === 'IPv6'; + }) + ); +}); -exports.indirectInstanceOf = function(obj, cls) { - if (obj instanceof cls) { return true; } - var clsChain = protoCtrChain(cls.prototype); - var objChain = protoCtrChain(obj); - return objChain.slice(-clsChain.length) === clsChain; +/* + * Check that when running a test with + * `$node --abort-on-uncaught-exception $file child` + * the process aborts. + */ +exports.childShouldThrowAndAbort = function () { + var testCmd = ''; + if (!exports.isWindows) { + // Do not create core files, as it can take a lot of disk space on + // continuous testing and developers' machines + testCmd += 'ulimit -c 0 && '; + } + testCmd += process.argv[0] + ' --abort-on-uncaught-exception '; + testCmd += process.argv[1] + ' child'; + var child = child_process.exec(testCmd); + child.on('exit', function onExit(exitCode, signal) { + var errMsg = 'Test should have aborted ' + ('but instead exited with exit code ' + exitCode) + (' and signal ' + signal); + assert(exports.nodeProcessAborted(exitCode, signal), errMsg); + }); }; - -exports.ddCommand = function(filename, kilobytes) { - if (process.platform === 'win32') { +exports.ddCommand = function (filename, kilobytes) { + if (exports.isWindows) { var p = path.resolve(exports.fixturesDir, 'create-file.js'); - return '"' + process.argv[0] + '" "' + p + '" "' + - filename + '" ' + (kilobytes * 1024); + return '"' + process.argv[0] + '" "' + p + '" "' + filename + '" ' + kilobytes * 1024; } else { return 'dd if=/dev/zero of="' + filename + '" bs=1024 count=' + kilobytes; } }; - -exports.spawnCat = function(options) { +exports.spawnPwd = function (options) { var spawn = require('child_process').spawn; - if (process.platform === 'win32') { - return spawn('more', [], options); + if (exports.isWindows) { + return spawn('cmd.exe', ['/d', '/c', 'cd'], options); } else { - return spawn('cat', [], options); + return spawn('pwd', [], options); } }; +exports.spawnSyncPwd = function (options) { + var spawnSync = require('child_process').spawnSync; -exports.spawnPwd = function(options) { - var spawn = require('child_process').spawn; - - if (process.platform === 'win32') { - return spawn('cmd.exe', ['/c', 'cd'], options); + if (exports.isWindows) { + return spawnSync('cmd.exe', ['/d', '/c', 'cd'], options); } else { - return spawn('pwd', [], options); + return spawnSync('pwd', [], options); } }; +exports.platformTimeout = function (ms) { + if (process.config.target_defaults.default_configuration === 'Debug') ms = 2 * ms; + + if (global.__coverage__) ms = 4 * ms; + + if (exports.isAix) return 2 * ms; // default localhost speed is slower on AIX + + if (process.arch !== 'arm') return ms; + + var armv = process.config.variables.arm_version; + + if (armv === '6') return 7 * ms; // ARMv6 + + if (armv === '7') return 2 * ms; // ARMv7 + + return ms; // ARMv8+ +}; + +var knownGlobals = [Buffer, clearImmediate, clearInterval, clearTimeout, console, constructor, // Enumerable in V8 3.21. +global, process, setImmediate, setInterval, setTimeout]; + +if (global.gc) { + knownGlobals.push(global.gc); +} + +if (global.DTRACE_HTTP_SERVER_RESPONSE) { + knownGlobals.push(DTRACE_HTTP_SERVER_RESPONSE); + knownGlobals.push(DTRACE_HTTP_SERVER_REQUEST); + knownGlobals.push(DTRACE_HTTP_CLIENT_RESPONSE); + knownGlobals.push(DTRACE_HTTP_CLIENT_REQUEST); + knownGlobals.push(DTRACE_NET_STREAM_END); + knownGlobals.push(DTRACE_NET_SERVER_CONNECTION); +} + +if (global.COUNTER_NET_SERVER_CONNECTION) { + knownGlobals.push(COUNTER_NET_SERVER_CONNECTION); + knownGlobals.push(COUNTER_NET_SERVER_CONNECTION_CLOSE); + knownGlobals.push(COUNTER_HTTP_SERVER_REQUEST); + knownGlobals.push(COUNTER_HTTP_SERVER_RESPONSE); + knownGlobals.push(COUNTER_HTTP_CLIENT_REQUEST); + knownGlobals.push(COUNTER_HTTP_CLIENT_RESPONSE); +} + +if (global.LTTNG_HTTP_SERVER_RESPONSE) { + knownGlobals.push(LTTNG_HTTP_SERVER_RESPONSE); + knownGlobals.push(LTTNG_HTTP_SERVER_REQUEST); + knownGlobals.push(LTTNG_HTTP_CLIENT_RESPONSE); + knownGlobals.push(LTTNG_HTTP_CLIENT_REQUEST); + knownGlobals.push(LTTNG_NET_STREAM_END); + knownGlobals.push(LTTNG_NET_SERVER_CONNECTION); +} + +if (global.ArrayBuffer) { + knownGlobals.push(ArrayBuffer); + knownGlobals.push(Int8Array); + knownGlobals.push(Uint8Array); + knownGlobals.push(Uint8ClampedArray); + knownGlobals.push(Int16Array); + knownGlobals.push(Uint16Array); + knownGlobals.push(Int32Array); + knownGlobals.push(Uint32Array); + knownGlobals.push(Float32Array); + knownGlobals.push(Float64Array); + knownGlobals.push(DataView); +} + +// Harmony features. +if (global.Proxy) { + knownGlobals.push(Proxy); +} + +if (global.Symbol) { + knownGlobals.push(Symbol); +} + +function allowGlobals() { + for (var _len = arguments.length, whitelist = Array(_len), _key = 0; _key < _len; _key++) { + whitelist[_key] = arguments[_key]; + } + + knownGlobals = knownGlobals.concat(whitelist); +} +exports.allowGlobals = allowGlobals; + +/**/ +if (typeof constructor == 'function') knownGlobals.push(constructor); +if (typeof DTRACE_NET_SOCKET_READ == 'function') knownGlobals.push(DTRACE_NET_SOCKET_READ); +if (typeof DTRACE_NET_SOCKET_WRITE == 'function') knownGlobals.push(DTRACE_NET_SOCKET_WRITE); +if (global.__coverage__) knownGlobals.push(__coverage__); +'core,__core-js_shared__,Promise,Map,Set,WeakMap,WeakSet,Reflect,System,asap,Observable,regeneratorRuntime,_babelPolyfill'.split(',').filter(function (item) { + return typeof global[item] !== undefined; +}).forEach(function (item) { + knownGlobals.push(global[item]); +}); /**/ + +function leakedGlobals() { + var leaked = []; + + for (var val in global) { + if (!knownGlobals.includes(global[val])) leaked.push(val); + }if (global.__coverage__) { + return leaked.filter(function (varname) { + return !/^(cov_|__cov)/.test(varname); + }); + } else { + return leaked; + } +} +exports.leakedGlobals = leakedGlobals; // Turn this off if the test should not check for global leaks. exports.globalCheck = true; -process.on('exit', function() { +process.on('exit', function () { if (!exports.globalCheck) return; - var knownGlobals = [setTimeout, - setInterval, - typeof setImmediate == 'undefined' ? null : setImmediate, - clearTimeout, - clearInterval, - typeof clearImmediate == 'undefined' ? null : clearImmediate, - console, - constructor, // Enumerable in V8 3.21. - Buffer, - process, - global].filter(Boolean); - - if (global.gc) { - knownGlobals.push(gc); - } - - if (global.DTRACE_HTTP_SERVER_RESPONSE) { - knownGlobals.push(DTRACE_HTTP_SERVER_RESPONSE); - knownGlobals.push(DTRACE_HTTP_SERVER_REQUEST); - knownGlobals.push(DTRACE_HTTP_CLIENT_RESPONSE); - knownGlobals.push(DTRACE_HTTP_CLIENT_REQUEST); - knownGlobals.push(DTRACE_NET_STREAM_END); - knownGlobals.push(DTRACE_NET_SERVER_CONNECTION); - knownGlobals.push(DTRACE_NET_SOCKET_READ); - knownGlobals.push(DTRACE_NET_SOCKET_WRITE); - } - if (global.COUNTER_NET_SERVER_CONNECTION) { - knownGlobals.push(COUNTER_NET_SERVER_CONNECTION); - knownGlobals.push(COUNTER_NET_SERVER_CONNECTION_CLOSE); - knownGlobals.push(COUNTER_HTTP_SERVER_REQUEST); - knownGlobals.push(COUNTER_HTTP_SERVER_RESPONSE); - knownGlobals.push(COUNTER_HTTP_CLIENT_REQUEST); - knownGlobals.push(COUNTER_HTTP_CLIENT_RESPONSE); - } - - if (global.ArrayBuffer) { - knownGlobals.push(ArrayBuffer); - knownGlobals.push(Int8Array); - knownGlobals.push(Uint8Array); - knownGlobals.push(Uint8ClampedArray); - knownGlobals.push(Int16Array); - knownGlobals.push(Uint16Array); - knownGlobals.push(Int32Array); - knownGlobals.push(Uint32Array); - knownGlobals.push(Float32Array); - knownGlobals.push(Float64Array); - knownGlobals.push(DataView); - } - - for (var x in global) { - var found = false; - - for (var y in knownGlobals) { - if (global[x] === knownGlobals[y]) { - found = true; - break; - } - } - - if (!found) { - console.error('Unknown global: %s', x); - assert.ok(false, 'Unknown global found'); - } + var leaked = leakedGlobals(); + if (leaked.length > 0) { + fail('Unexpected global(s) found: ' + leaked.join(', ')); } }); - var mustCallChecks = []; - function runCallChecks(exitCode) { if (exitCode !== 0) return; - var failed = mustCallChecks.filter(function(context) { + var failed = mustCallChecks.filter(function (context) { return context.actual !== context.expected; }); - failed.forEach(function(context) { - console.log('Mismatched %s function calls. Expected %d, actual %d.', - context.name, - context.expected, - context.actual); + failed.forEach(function (context) { + console.log('Mismatched %s function calls. Expected %d, actual %d.', context.name, context.expected, context.actual); console.log(context.stack.split('\n').slice(2).join('\n')); }); if (failed.length) process.exit(1); } - -exports.mustCall = function(fn, expected) { - if (typeof expected !== 'number') expected = 1; +exports.mustCall = function (fn, expected) { + if (expected === undefined) expected = 1;else if (typeof expected !== 'number') throw new TypeError('Invalid expected value: ' + expected); var context = { expected: expected, actual: 0, - stack: (new Error).stack, + stack: new Error().stack, name: fn.name || '' }; @@ -196,8 +387,197 @@ exports.mustCall = function(fn, expected) { mustCallChecks.push(context); - return function() { + return function () { context.actual++; return fn.apply(this, arguments); }; }; + +exports.hasMultiLocalhost = function hasMultiLocalhost() { + var TCP = process.binding('tcp_wrap').TCP; + var t = new TCP(); + var ret = t.bind('127.0.0.2', exports.PORT); + t.close(); + return ret === 0; +}; + +exports.fileExists = function (pathname) { + try { + fs.accessSync(pathname); + return true; + } catch (err) { + return false; + } +}; + +exports.canCreateSymLink = function () { + // On Windows, creating symlinks requires admin privileges. + // We'll only try to run symlink test if we have enough privileges. + // On other platforms, creating symlinks shouldn't need admin privileges + if (exports.isWindows) { + // whoami.exe needs to be the one from System32 + // If unix tools are in the path, they can shadow the one we want, + // so use the full path while executing whoami + var whoamiPath = path.join(process.env['SystemRoot'], 'System32', 'whoami.exe'); + + var err = false; + var output = ''; + + try { + output = execSync(whoamiPath + ' /priv', { timout: 1000 }); + } catch (e) { + err = true; + } finally { + if (err || !output.includes('SeCreateSymbolicLinkPrivilege')) { + return false; + } + } + } + + return true; +}; + +function fail(msg) { + assert.fail(null, null, msg); +} +exports.fail = fail; + +exports.mustNotCall = function (msg) { + return function mustNotCall() { + fail(msg || 'function should not have been called'); + }; +}; + +exports.skip = function (msg) { + console.log('1..0 # Skipped: ' + msg); +}; + +// A stream to push an array into a REPL +function ArrayStream() { + this.run = function (data) { + var _this = this; + + data.forEach(function (line) { + _this.emit('data', line + '\n'); + }); + }; +} + +util.inherits(ArrayStream, stream.Stream); +exports.ArrayStream = ArrayStream; +ArrayStream.prototype.readable = true; +ArrayStream.prototype.writable = true; +ArrayStream.prototype.pause = function () {}; +ArrayStream.prototype.resume = function () {}; +ArrayStream.prototype.write = function () {}; + +// Returns true if the exit code "exitCode" and/or signal name "signal" +// represent the exit code and/or signal name of a node process that aborted, +// false otherwise. +exports.nodeProcessAborted = function nodeProcessAborted(exitCode, signal) { + // Depending on the compiler used, node will exit with either + // exit code 132 (SIGILL), 133 (SIGTRAP) or 134 (SIGABRT). + var expectedExitCodes = [132, 133, 134]; + + // On platforms using KSH as the default shell (like SmartOS), + // when a process aborts, KSH exits with an exit code that is + // greater than 256, and thus the exit code emitted with the 'exit' + // event is null and the signal is set to either SIGILL, SIGTRAP, + // or SIGABRT (depending on the compiler). + var expectedSignals = ['SIGILL', 'SIGTRAP', 'SIGABRT']; + + // On Windows, v8's base::OS::Abort triggers an access violation, + // which corresponds to exit code 3221225477 (0xC0000005) + if (exports.isWindows) expectedExitCodes = [3221225477]; + + // When using --abort-on-uncaught-exception, V8 will use + // base::OS::Abort to terminate the process. + // Depending on the compiler used, the shell or other aspects of + // the platform used to build the node binary, this will actually + // make V8 exit by aborting or by raising a signal. In any case, + // one of them (exit code or signal) needs to be set to one of + // the expected exit codes or signals. + if (signal !== null) { + return expectedSignals.includes(signal); + } else { + return expectedExitCodes.includes(exitCode); + } +}; + +exports.busyLoop = function busyLoop(time) { + var startTime = Timer.now(); + var stopTime = startTime + time; + while (Timer.now() < stopTime) {} +}; + +exports.isAlive = function isAlive(pid) { + try { + process.kill(pid, 'SIGCONT'); + return true; + } catch (e) { + return false; + } +}; + +exports.expectWarning = function (name, expected) { + if (typeof expected === 'string') expected = [expected]; + process.on('warning', exports.mustCall(function (warning) { + assert.strictEqual(warning.name, name); + assert.ok(expected.includes(warning.message), 'unexpected error message: "' + warning.message + '"'); + // Remove a warning message after it is seen so that we guarantee that we + // get each message only once. + expected.splice(expected.indexOf(warning.message), 1); + }, expected.length)); +}; + +Object.defineProperty(exports, 'hasIntl', { + get: function () { + return process.binding('config').hasIntl; + } +}); + +// https://github.com/w3c/testharness.js/blob/master/testharness.js +exports.WPT = { + test: function (fn, desc) { + try { + fn(); + } catch (err) { + if (err instanceof Error) err.message = 'In ' + desc + ':\n ' + err.message; + throw err; + } + }, + assert_equals: assert.strictEqual, + assert_true: function (value, message) { + return assert.strictEqual(value, true, message); + }, + assert_false: function (value, message) { + return assert.strictEqual(value, false, message); + }, + assert_throws: function (code, func, desc) { + assert.throws(func, function (err) { + return typeof err === 'object' && 'name' in err && err.name === code.name; + }, desc); + }, + assert_array_equals: assert.deepStrictEqual, + assert_unreached: function (desc) { + assert.fail(undefined, undefined, 'Reached unreachable code: ' + desc); + } +}; + +// Useful for testing expected internal/error objects +exports.expectsError = function expectsError(_ref) { + var code = _ref.code, + type = _ref.type, + message = _ref.message; + + return function (error) { + assert.strictEqual(error.code, code); + if (type !== undefined) assert(error instanceof type, error + ' is not the expected type ' + type); + if (message instanceof RegExp) { + assert(message.test(error.message), error.message + ' does not match ' + message); + } else if (typeof message === 'string') { + assert.strictEqual(error.message, message); + } + return true; + }; +}; \ No newline at end of file diff --git a/test/parallel/test-string-decoder-end.js b/test/parallel/test-string-decoder-end.js new file mode 100644 index 0000000..b82c3ca --- /dev/null +++ b/test/parallel/test-string-decoder-end.js @@ -0,0 +1,60 @@ +'use strict'; + +var bufferShim = require('buffer-shims'); +// verify that the string decoder works getting 1 byte at a time, +// the whole buffer at once, and that both match the .toString(enc) +// result of the entire buffer. + +require('../common'); +var assert = require('assert'); +var SD = require('../../').StringDecoder; +var encodings = ['base64', 'hex', 'utf8', 'utf16le', 'ucs2']; + +var bufs = ['☃💩', 'asdf'].map(function (b) { + return bufferShim.from(b); +}); + +// also test just arbitrary bytes from 0-15. +for (var i = 1; i <= 16; i++) { + var bytes = '.'.repeat(i - 1).split('.').map(function (_, j) { + return j + 0x78; + }); + bufs.push(bufferShim.from(bytes)); +} + +encodings.forEach(testEncoding); + +console.log('ok'); + +function testEncoding(encoding) { + bufs.forEach(function (buf) { + testBuf(encoding, buf); + }); +} + +function testBuf(encoding, buf) { + console.error('# %s', encoding, buf); + + // write one byte at a time. + var s = new SD(encoding); + var res1 = ''; + for (var _i = 0; _i < buf.length; _i++) { + res1 += s.write(buf.slice(_i, _i + 1)); + } + res1 += s.end(); + + // write the whole buffer at once. + var res2 = ''; + s = new SD(encoding); + res2 += s.write(buf); + res2 += s.end(); + + // .toString() on the buffer + var res3 = buf.toString(encoding); + + console.log('expect=%j', res3); + console.log('res1=%j', res1); + console.log('res2=%j', res2); + assert.strictEqual(res1, res3, 'one byte at a time should match toString'); + assert.strictEqual(res2, res3, 'all bytes at once should match toString'); +} \ No newline at end of file diff --git a/test/parallel/test-string-decoder.js b/test/parallel/test-string-decoder.js new file mode 100644 index 0000000..38e21d2 --- /dev/null +++ b/test/parallel/test-string-decoder.js @@ -0,0 +1,173 @@ +'use strict'; + +var bufferShim = require('buffer-shims'); +require('../common'); +var assert = require('assert'); +var inspect = require('util').inspect; +var StringDecoder = require('../../').StringDecoder; + +// Test default encoding +var decoder = new StringDecoder(); +assert.strictEqual(decoder.encoding, 'utf8'); + +process.stdout.write('scanning '); + +// UTF-8 +test('utf-8', bufferShim.from('$', 'utf-8'), '$'); +test('utf-8', bufferShim.from('¢', 'utf-8'), '¢'); +test('utf-8', bufferShim.from('€', 'utf-8'), '€'); +test('utf-8', bufferShim.from('𤭢', 'utf-8'), '𤭢'); +// A mixed ascii and non-ascii string +// Test stolen from deps/v8/test/cctest/test-strings.cc +// U+02E4 -> CB A4 +// U+0064 -> 64 +// U+12E4 -> E1 8B A4 +// U+0030 -> 30 +// U+3045 -> E3 81 85 +test('utf-8', bufferShim.from([0xCB, 0xA4, 0x64, 0xE1, 0x8B, 0xA4, 0x30, 0xE3, 0x81, 0x85]), '\u02e4\u0064\u12e4\u0030\u3045'); + +// Some invalid input, known to have caused trouble with chunking +// in https://github.com/nodejs/node/pull/7310#issuecomment-226445923 +// 00: |00000000 ASCII +// 41: |01000001 ASCII +// B8: 10|111000 continuation +// CC: 110|01100 two-byte head +// E2: 1110|0010 three-byte head +// F0: 11110|000 four-byte head +// F1: 11110|001'another four-byte head +// FB: 111110|11 "five-byte head", not UTF-8 +test('utf-8', bufferShim.from('C9B5A941', 'hex'), '\u0275\ufffdA'); +test('utf-8', bufferShim.from('E2', 'hex'), '\ufffd'); +test('utf-8', bufferShim.from('E241', 'hex'), '\ufffdA'); +test('utf-8', bufferShim.from('CCCCB8', 'hex'), '\ufffd\u0338'); +test('utf-8', bufferShim.from('F0B841', 'hex'), '\ufffd\ufffdA'); +test('utf-8', bufferShim.from('F1CCB8', 'hex'), '\ufffd\u0338'); +test('utf-8', bufferShim.from('F0FB00', 'hex'), '\ufffd\ufffd\0'); +test('utf-8', bufferShim.from('CCE2B8B8', 'hex'), '\ufffd\u2e38'); +test('utf-8', bufferShim.from('E2B8CCB8', 'hex'), '\ufffd\ufffd\u0338'); +test('utf-8', bufferShim.from('E2FBCC01', 'hex'), '\ufffd\ufffd\ufffd\u0001'); +test('utf-8', bufferShim.from('CCB8CDB9', 'hex'), '\u0338\u0379'); + +// UCS-2 +test('ucs2', bufferShim.from('ababc', 'ucs2'), 'ababc'); + +// UTF-16LE +test('utf16le', bufferShim.from('3DD84DDC', 'hex'), '\ud83d\udc4d'); // thumbs up + +console.log(' crayon!'); + +// Additional UTF-8 tests +decoder = new StringDecoder('utf8'); +assert.strictEqual(decoder.write(bufferShim.from('E1', 'hex')), ''); +assert.strictEqual(decoder.end(), '\ufffd'); + +decoder = new StringDecoder('utf8'); +assert.strictEqual(decoder.write(bufferShim.from('E18B', 'hex')), ''); +assert.strictEqual(decoder.end(), '\ufffd\ufffd'); + +decoder = new StringDecoder('utf8'); +assert.strictEqual(decoder.write(bufferShim.from('\ufffd')), '\ufffd'); +assert.strictEqual(decoder.end(), ''); + +decoder = new StringDecoder('utf8'); +assert.strictEqual(decoder.write(bufferShim.from('\ufffd\ufffd\ufffd')), '\ufffd\ufffd\ufffd'); +assert.strictEqual(decoder.end(), ''); + +decoder = new StringDecoder('utf8'); +assert.strictEqual(decoder.write(bufferShim.from('EFBFBDE2', 'hex')), '\ufffd'); +assert.strictEqual(decoder.end(), '\ufffd'); + +decoder = new StringDecoder('utf8'); +assert.strictEqual(decoder.write(bufferShim.from('F1', 'hex')), ''); +assert.strictEqual(decoder.write(bufferShim.from('41F2', 'hex')), '\ufffdA'); +assert.strictEqual(decoder.end(), '\ufffd'); + +// Additional utf8Text test +decoder = new StringDecoder('utf8'); +assert.strictEqual(decoder.text(bufferShim.from([0x41]), 2), ''); + +// Additional UTF-16LE surrogate pair tests +decoder = new StringDecoder('utf16le'); +assert.strictEqual(decoder.write(bufferShim.from('3DD8', 'hex')), ''); +assert.strictEqual(decoder.write(bufferShim.from('4D', 'hex')), ''); +assert.strictEqual(decoder.write(bufferShim.from('DC', 'hex')), '\ud83d\udc4d'); +assert.strictEqual(decoder.end(), ''); + +decoder = new StringDecoder('utf16le'); +assert.strictEqual(decoder.write(bufferShim.from('3DD8', 'hex')), ''); +assert.strictEqual(decoder.end(), '\ud83d'); + +decoder = new StringDecoder('utf16le'); +assert.strictEqual(decoder.write(bufferShim.from('3DD8', 'hex')), ''); +assert.strictEqual(decoder.write(bufferShim.from('4D', 'hex')), ''); +assert.strictEqual(decoder.end(), '\ud83d'); + +assert.throws(function () { + new StringDecoder(1); +}, /^Error: Unknown encoding: 1$/); + +assert.throws(function () { + new StringDecoder('test'); +}, /^Error: Unknown encoding: test$/); + +// test verifies that StringDecoder will correctly decode the given input +// buffer with the given encoding to the expected output. It will attempt all +// possible ways to write() the input buffer, see writeSequences(). The +// singleSequence allows for easy debugging of a specific sequence which is +// useful in case of test failures. +function test(encoding, input, expected, singleSequence) { + var sequences = void 0; + if (!singleSequence) { + sequences = writeSequences(input.length); + } else { + sequences = [singleSequence]; + } + sequences.forEach(function (sequence) { + var decoder = new StringDecoder(encoding); + var output = ''; + sequence.forEach(function (write) { + output += decoder.write(input.slice(write[0], write[1])); + }); + output += decoder.end(); + process.stdout.write('.'); + if (output !== expected) { + var message = 'Expected "' + unicodeEscape(expected) + '", ' + 'but got "' + unicodeEscape(output) + '"\n' + 'input: ' + input.toString('hex').match(/.{2}/g) + '\n' + 'Write sequence: ' + JSON.stringify(sequence) + '\n' + 'Full Decoder State: ' + inspect(decoder); + assert.fail(output, expected, message); + } + }); +} + +// unicodeEscape prints the str contents as unicode escape codes. +function unicodeEscape(str) { + var r = ''; + for (var i = 0; i < str.length; i++) { + r += '\\u' + str.charCodeAt(i).toString(16); + } + return r; +} + +// writeSequences returns an array of arrays that describes all possible ways a +// buffer of the given length could be split up and passed to sequential write +// calls. +// +// e.G. writeSequences(3) will return: [ +// [ [ 0, 3 ] ], +// [ [ 0, 2 ], [ 2, 3 ] ], +// [ [ 0, 1 ], [ 1, 3 ] ], +// [ [ 0, 1 ], [ 1, 2 ], [ 2, 3 ] ] +// ] +function writeSequences(length, start, sequence) { + if (start === undefined) { + start = 0; + sequence = []; + } else if (start === length) { + return [sequence]; + } + var sequences = []; + for (var end = length; end > start; end--) { + var subSequence = sequence.concat([[start, end]]); + var subSequences = writeSequences(length, end, subSequence, sequences); + sequences = sequences.concat(subSequences); + } + return sequences; +} \ No newline at end of file diff --git a/test/simple/test-string-decoder-end.js b/test/simple/test-string-decoder-end.js deleted file mode 100644 index 869a411..0000000 --- a/test/simple/test-string-decoder-end.js +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright Joyent, Inc. and other Node contributors. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to permit -// persons to whom the Software is furnished to do so, subject to the -// following conditions: -// -// The above copyright notice and this permission notice shall be included -// in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -// USE OR OTHER DEALINGS IN THE SOFTWARE. - -// verify that the string decoder works getting 1 byte at a time, -// the whole buffer at once, and that both match the .toString(enc) -// result of the entire buffer. - -var assert = require('assert'); -var SD = require('../../').StringDecoder; -var encodings = ['base64', 'hex', 'utf8', 'utf16le', 'ucs2']; - -var bufs = [ '☃💩', 'asdf' ].map(function(b) { - return new Buffer(b); -}); - -// also test just arbitrary bytes from 0-15. -for (var i = 1; i <= 16; i++) { - var bytes = new Array(i).join('.').split('.').map(function(_, j) { - return j + 0x78; - }); - bufs.push(new Buffer(bytes)); -} - -encodings.forEach(testEncoding); - -console.log('ok'); - -function testEncoding(encoding) { - bufs.forEach(function(buf) { - testBuf(encoding, buf); - }); -} - -function testBuf(encoding, buf) { - console.error('# %s', encoding, buf); - - // write one byte at a time. - var s = new SD(encoding); - var res1 = ''; - for (var i = 0; i < buf.length; i++) { - res1 += s.write(buf.slice(i, i + 1)); - } - res1 += s.end(); - - // write the whole buffer at once. - var res2 = ''; - var s = new SD(encoding); - res2 += s.write(buf); - res2 += s.end(); - - // .toString() on the buffer - var res3 = buf.toString(encoding); - - console.log('expect=%j', res3); - assert.equal(res1, res3, 'one byte at a time should match toString'); - assert.equal(res2, res3, 'all bytes at once should match toString'); -} diff --git a/test/simple/test-string-decoder.js b/test/simple/test-string-decoder.js deleted file mode 100644 index 7f69f7e..0000000 --- a/test/simple/test-string-decoder.js +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright Joyent, Inc. and other Node contributors. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to permit -// persons to whom the Software is furnished to do so, subject to the -// following conditions: -// -// The above copyright notice and this permission notice shall be included -// in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -// USE OR OTHER DEALINGS IN THE SOFTWARE. - -var common = require('../common'); -var assert = require('assert'); -var StringDecoder = require('../../').StringDecoder; -var decoder = new StringDecoder('utf8'); - - - -var buffer = new Buffer('$'); -assert.deepEqual('$', decoder.write(buffer)); - -buffer = new Buffer('¢'); -assert.deepEqual('', decoder.write(buffer.slice(0, 1))); -assert.deepEqual('¢', decoder.write(buffer.slice(1, 2))); - -buffer = new Buffer('€'); -assert.deepEqual('', decoder.write(buffer.slice(0, 1))); -assert.deepEqual('', decoder.write(buffer.slice(1, 2))); -assert.deepEqual('€', decoder.write(buffer.slice(2, 3))); - -buffer = new Buffer([0xF0, 0xA4, 0xAD, 0xA2]); -var s = ''; -s += decoder.write(buffer.slice(0, 1)); -s += decoder.write(buffer.slice(1, 2)); -s += decoder.write(buffer.slice(2, 3)); -s += decoder.write(buffer.slice(3, 4)); -assert.ok(s.length > 0); - -// CESU-8 -buffer = new Buffer('EDA0BDEDB18D', 'hex'); // THUMBS UP SIGN (in CESU-8) -var s = ''; -s += decoder.write(buffer.slice(0, 1)); -s += decoder.write(buffer.slice(1, 2)); -s += decoder.write(buffer.slice(2, 3)); // complete lead surrogate -assert.equal(s, ''); -s += decoder.write(buffer.slice(3, 4)); -s += decoder.write(buffer.slice(4, 5)); -s += decoder.write(buffer.slice(5, 6)); // complete trail surrogate -assert.equal(s, '\uD83D\uDC4D'); // THUMBS UP SIGN (in UTF-16) - -var s = ''; -s += decoder.write(buffer.slice(0, 2)); -s += decoder.write(buffer.slice(2, 4)); // complete lead surrogate -assert.equal(s, ''); -s += decoder.write(buffer.slice(4, 6)); // complete trail surrogate -assert.equal(s, '\uD83D\uDC4D'); // THUMBS UP SIGN (in UTF-16) - -var s = ''; -s += decoder.write(buffer.slice(0, 3)); // complete lead surrogate -assert.equal(s, ''); -s += decoder.write(buffer.slice(3, 6)); // complete trail surrogate -assert.equal(s, '\uD83D\uDC4D'); // THUMBS UP SIGN (in UTF-16) - -var s = ''; -s += decoder.write(buffer.slice(0, 4)); // complete lead surrogate -assert.equal(s, ''); -s += decoder.write(buffer.slice(4, 5)); -s += decoder.write(buffer.slice(5, 6)); // complete trail surrogate -assert.equal(s, '\uD83D\uDC4D'); // THUMBS UP SIGN (in UTF-16) - -var s = ''; -s += decoder.write(buffer.slice(0, 5)); // complete lead surrogate -assert.equal(s, ''); -s += decoder.write(buffer.slice(5, 6)); // complete trail surrogate -assert.equal(s, '\uD83D\uDC4D'); // THUMBS UP SIGN (in UTF-16) - -var s = ''; -s += decoder.write(buffer.slice(0, 6)); -assert.equal(s, '\uD83D\uDC4D'); // THUMBS UP SIGN (in UTF-16) - - -// UCS-2 -decoder = new StringDecoder('ucs2'); -buffer = new Buffer('ab', 'ucs2'); -assert.equal(decoder.write(buffer), 'ab'); // 2 complete chars -buffer = new Buffer('abc', 'ucs2'); -assert.equal(decoder.write(buffer.slice(0, 3)), 'a'); // 'a' and first of 'b' -assert.equal(decoder.write(buffer.slice(3, 6)), 'bc'); // second of 'b' and 'c' - - -// UTF-16LE -buffer = new Buffer('3DD84DDC', 'hex'); // THUMBS UP SIGN (in CESU-8) -var s = ''; -s += decoder.write(buffer.slice(0, 1)); -s += decoder.write(buffer.slice(1, 2)); // complete lead surrogate -assert.equal(s, ''); -s += decoder.write(buffer.slice(2, 3)); -s += decoder.write(buffer.slice(3, 4)); // complete trail surrogate -assert.equal(s, '\uD83D\uDC4D'); // THUMBS UP SIGN (in UTF-16) - -var s = ''; -s += decoder.write(buffer.slice(0, 2)); // complete lead surrogate -assert.equal(s, ''); -s += decoder.write(buffer.slice(2, 4)); // complete trail surrogate -assert.equal(s, '\uD83D\uDC4D'); // THUMBS UP SIGN (in UTF-16) - -var s = ''; -s += decoder.write(buffer.slice(0, 3)); // complete lead surrogate -assert.equal(s, ''); -s += decoder.write(buffer.slice(3, 4)); // complete trail surrogate -assert.equal(s, '\uD83D\uDC4D'); // THUMBS UP SIGN (in UTF-16) - -var s = ''; -s += decoder.write(buffer.slice(0, 4)); -assert.equal(s, '\uD83D\uDC4D'); // THUMBS UP SIGN (in UTF-16) - - -// A mixed ascii and non-ascii string -// Test stolen from deps/v8/test/cctest/test-strings.cc -// U+02E4 -> CB A4 -// U+0064 -> 64 -// U+12E4 -> E1 8B A4 -// U+0030 -> 30 -// U+3045 -> E3 81 85 -var expected = '\u02e4\u0064\u12e4\u0030\u3045'; -var buffer = new Buffer([0xCB, 0xA4, 0x64, 0xE1, 0x8B, 0xA4, - 0x30, 0xE3, 0x81, 0x85]); -var charLengths = [0, 0, 1, 2, 2, 2, 3, 4, 4, 4, 5, 5]; - -// Split the buffer into 3 segments -// |----|------|-------| -// 0 i j buffer.length -// Scan through every possible 3 segment combination -// and make sure that the string is always parsed. -common.print('scanning '); -for (var j = 2; j < buffer.length; j++) { - for (var i = 1; i < j; i++) { - var decoder = new StringDecoder('utf8'); - - var sum = decoder.write(buffer.slice(0, i)); - - // just check that we've received the right amount - // after the first write - assert.equal(charLengths[i], sum.length); - - sum += decoder.write(buffer.slice(i, j)); - sum += decoder.write(buffer.slice(j, buffer.length)); - assert.equal(expected, sum); - common.print('.'); - } -} -console.log(' crayon!'); -