Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

src: make base64 decoding 10-15% faster #2193

Merged
merged 2 commits into from
Jul 25, 2015
Merged

src: make base64 decoding 10-15% faster #2193

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8fd3ce1
src: make base64 decoding 50% faster
bnoordhuis Jul 15, 2015
ac70bc8
test: fix valgrind uninitialized memory warning
bnoordhuis Jul 16, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions benchmark/buffers/buffer-base64-decode.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
var assert = require('assert');
var common = require('../common.js');

var bench = common.createBenchmark(main, {});

function main(conf) {
for (var s = 'abcd'; s.length < 32 << 20; s += s);
s.match(/./); // Flatten string.
assert.equal(s.length % 4, 0);
var b = Buffer(s.length / 4 * 3);
b.write(s, 0, s.length, 'base64');
bench.start();
for (var i = 0; i < 32; i += 1) b.base64Write(s, 0, s.length);
bench.end(32);
}
115 changes: 68 additions & 47 deletions src/string_bytes.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ size_t base64_decoded_size(const TypeName* src, size_t size) {


// supports regular and URL-safe base64
static const int unbase64_table[] =
static const int8_t unbase64_table[] =
{ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -2, -1, -1, -2, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-2, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, 62, -1, 63,
Expand All @@ -150,62 +150,83 @@ static const int unbase64_table[] =
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1
};
#define unbase64(x) unbase64_table[(uint8_t)(x)]
#define unbase64(x) \
static_cast<uint8_t>(unbase64_table[static_cast<uint8_t>(x)])


template <typename TypeName>
size_t base64_decode(char* buf,
size_t len,
const TypeName* src,
const size_t srcLen) {
char a, b, c, d;
char* dst = buf;
char* dstEnd = buf + len;
const TypeName* srcEnd = src + srcLen;

while (src < srcEnd && dst < dstEnd) {
int remaining = srcEnd - src;

while (unbase64(*src) < 0 && src < srcEnd)
src++, remaining--;
if (remaining == 0 || *src == '=')
break;
a = unbase64(*src++);

while (unbase64(*src) < 0 && src < srcEnd)
src++, remaining--;
if (remaining <= 1 || *src == '=')
break;
b = unbase64(*src++);

*dst++ = (a << 2) | ((b & 0x30) >> 4);
if (dst == dstEnd)
break;

while (unbase64(*src) < 0 && src < srcEnd)
src++, remaining--;
if (remaining <= 2 || *src == '=')
break;
c = unbase64(*src++);
size_t base64_decode_slow(char* dst, size_t dstlen,
const TypeName* src, size_t srclen) {
uint8_t hi;
uint8_t lo;
size_t i = 0;
size_t k = 0;
for (;;) {
#define V(expr) \
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It could be just me, but defining it outside the loop might improve the readability.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Convention in the source tree is to define V-style codegen macros right before use.

while (i < srclen) { \
const uint8_t c = src[i]; \
lo = unbase64(c); \
i += 1; \
if (lo < 64) \
break; /* Legal character. */ \
if (c == '=') \
return k; \
} \
expr; \
if (i >= srclen) \
return k; \
if (k >= dstlen) \
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Style nit: Can this be combined with the previous if?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wrote it like this because personally, I find separate statements easier to read than boolean combinators. With the later, it's just easier to mix up the logic.

return k; \
hi = lo;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I may be totally missing something. But to me, it looks like hi and lo would always be the same and hi looks redundant.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have to admit the codegen macro kind of obscures it but the expr; statement at line 175 is where hi is the previous byte and lo the current one. That's also why the first V macro call has no expression, because hi is still uninitialized at that point.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh yeah. Understood now. Thanks for clarifying :-)

V(/* Nothing. */);
V(dst[k++] = ((hi & 0x3F) << 2) | ((lo & 0x30) >> 4));
V(dst[k++] = ((hi & 0x0F) << 4) | ((lo & 0x3C) >> 2));
V(dst[k++] = ((hi & 0x03) << 6) | ((lo & 0x3F) >> 0));
#undef V
}
UNREACHABLE();
}

*dst++ = ((b & 0x0F) << 4) | ((c & 0x3C) >> 2);
if (dst == dstEnd)
break;

while (unbase64(*src) < 0 && src < srcEnd)
src++, remaining--;
if (remaining <= 3 || *src == '=')
template <typename TypeName>
size_t base64_decode_fast(char* const dst, const size_t dstlen,
const TypeName* const src, const size_t srclen,
const size_t decoded_size) {
const size_t available = dstlen < decoded_size ? dstlen : decoded_size;
const size_t max_i = srclen / 4 * 4;
const size_t max_k = available / 3 * 3;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hope we are doing the integer floor operation in these two statements.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what do you mean? I can't tell if you're making a suggestion or questioning the code.

size_t i = 0;
size_t k = 0;
while (i < max_i && k < max_k) {
const uint32_t v =
unbase64(src[i + 0]) << 24 |
unbase64(src[i + 1]) << 16 |
unbase64(src[i + 2]) << 8 |
unbase64(src[i + 3]);
// If MSB is set, input contains whitespace or is not valid base64.
if (v & 0x80808080) {
break;
d = unbase64(*src++);

*dst++ = ((c & 0x03) << 6) | (d & 0x3F);
}
dst[k + 0] = ((v >> 22) & 0xFC) | ((v >> 20) & 0x03);
dst[k + 1] = ((v >> 12) & 0xF0) | ((v >> 10) & 0x0F);
dst[k + 2] = ((v >> 2) & 0xC0) | ((v >> 0) & 0x3F);
i += 4;
k += 3;
}

return dst - buf;
if (i < srclen && k < dstlen) {
return k + base64_decode_slow(dst + k, dstlen - k, src + i, srclen - i);
}
return k;
}


//// HEX ////
template <typename TypeName>
size_t base64_decode(char* const dst, const size_t dstlen,
const TypeName* const src, const size_t srclen) {
const size_t decoded_size = base64_decoded_size(src, srclen);
return base64_decode_fast(dst, dstlen, src, srclen, decoded_size);
}


template <typename TypeName>
unsigned hex2bin(TypeName c) {
Expand Down
4 changes: 3 additions & 1 deletion test/parallel/test-buffer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -802,7 +802,9 @@ assert.equal(buf[4], 0);

// Check for fractional length args, junk length args, etc.
// https://github.com/joyent/node/issues/1758
Buffer(3.3).toString(); // throws bad argument error in commit 43cb4ec

// Call .fill() first, stops valgrind warning about uninitialized memory reads.
Buffer(3.3).fill().toString(); // throws bad argument error in commit 43cb4ec
assert.equal(Buffer(-1).length, 0);
assert.equal(Buffer(NaN).length, 0);
assert.equal(Buffer(3.3).length, 3);
Expand Down