Skip to content

Commit

Permalink
test: update WPT encoding tests
Browse files Browse the repository at this point in the history
PR-URL: #43958
Reviewed-By: Luigi Pinca <[email protected]>
Reviewed-By: Feng Yu <[email protected]>
Reviewed-By: Darshan Sen <[email protected]>
  • Loading branch information
cola119 authored and danielleadams committed Jul 26, 2022
1 parent 51cb0d4 commit 198cf59
Show file tree
Hide file tree
Showing 15 changed files with 259 additions and 22 deletions.
2 changes: 1 addition & 1 deletion test/fixtures/wpt/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ Last update:
- console: https://github.com/web-platform-tests/wpt/tree/3b1f72e99a/console
- dom/abort: https://github.com/web-platform-tests/wpt/tree/c49cafb491/dom/abort
- dom/events: https://github.com/web-platform-tests/wpt/tree/f8821adb28/dom/events
- encoding: https://github.com/web-platform-tests/wpt/tree/35f70910d3/encoding
- encoding: https://github.com/web-platform-tests/wpt/tree/c1b24fce6e/encoding
- FileAPI: https://github.com/web-platform-tests/wpt/tree/3b279420d4/FileAPI
- hr-time: https://github.com/web-platform-tests/wpt/tree/9910784394/hr-time
- html/webappapis/atob: https://github.com/web-platform-tests/wpt/tree/f267e1dca6/html/webappapis/atob
Expand Down
25 changes: 14 additions & 11 deletions test/fixtures/wpt/encoding/encodeInto.any.js
Original file line number Diff line number Diff line change
Expand Up @@ -120,19 +120,22 @@
});
});

[DataView,
Int8Array,
Int16Array,
Int32Array,
Uint16Array,
Uint32Array,
Uint8ClampedArray,
Float32Array,
Float64Array].forEach(view => {
["DataView",
"Int8Array",
"Int16Array",
"Int32Array",
"Uint16Array",
"Uint32Array",
"Uint8ClampedArray",
"BigInt64Array",
"BigUint64Array",
"Float32Array",
"Float64Array"].forEach(type => {
["ArrayBuffer", "SharedArrayBuffer"].forEach((arrayBufferOrSharedArrayBuffer) => {
test(() => {
assert_throws_js(TypeError, () => new TextEncoder().encodeInto("", new view(createBuffer(arrayBufferOrSharedArrayBuffer, 0))));
}, "Invalid encodeInto() destination: " + view.name + ", backed by: " + arrayBufferOrSharedArrayBuffer);
const viewInstance = new self[type](createBuffer(arrayBufferOrSharedArrayBuffer, 0));
assert_throws_js(TypeError, () => new TextEncoder().encodeInto("", viewInstance));
}, "Invalid encodeInto() destination: " + type + ", backed by: " + arrayBufferOrSharedArrayBuffer);
});
});

Expand Down
2 changes: 2 additions & 0 deletions test/fixtures/wpt/encoding/encodeInto.any.js.headers
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cross-Origin-Opener-Policy: same-origin
Cross-Origin-Embedder-Policy: require-corp
2 changes: 2 additions & 0 deletions test/fixtures/wpt/encoding/idlharness-shadowrealm.window.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
// META: script=/resources/idlharness-shadowrealm.js
idl_test_shadowrealm(["encoding"], ["streams"]);
14 changes: 14 additions & 0 deletions test/fixtures/wpt/encoding/sharedarraybuffer.https.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
<!DOCTYPE html>
<script src="/resources/testharness.js"></script>
<script src="/resources/testharnessreport.js"></script>
<script>

test(() => {
const decoder = new TextDecoder('utf-8');
const buffer = new SharedArrayBuffer(4);
assert_throws_js(TypeError, () => {
decoder.decode(new Uint8Array(buffer));
}, 'constructing TextDecoder with SharedArrayBuffer view should throw');
}, 'decoding SharedArrayBuffer');

</script>
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cross-Origin-Opener-Policy:same-origin
Cross-Origin-Embedder-Policy:require-corp
107 changes: 107 additions & 0 deletions test/fixtures/wpt/encoding/single-byte-decoder.window.js

Large diffs are not rendered by default.

11 changes: 10 additions & 1 deletion test/fixtures/wpt/encoding/streams/decode-utf8.any.js
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,16 @@
const array = await readableStreamToArray(output);
assert_array_equals(array, [expectedOutputString],
'the output should be in one chunk');
}, 'a trailing empty chunk should be ignored- ' + arrayBufferOrSharedArrayBuffer);
}, 'a trailing empty chunk should be ignored - ' + arrayBufferOrSharedArrayBuffer);

promise_test(async () => {
const chunk = new Uint8Array(createBuffer(arrayBufferOrSharedArrayBuffer, 3));
chunk.set([0xF0, 0x9F, 0x92]);
const input = readableStreamFromArray([chunk]);
const output = input.pipeThrough(new TextDecoderStream());
const array = await readableStreamToArray(output);
assert_array_equals(array, ['\uFFFD']);
}, 'UTF-8 EOF handling - ' + arrayBufferOrSharedArrayBuffer);
});

promise_test(async () => {
Expand Down
2 changes: 2 additions & 0 deletions test/fixtures/wpt/encoding/streams/decode-utf8.any.js.headers
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cross-Origin-Opener-Policy: same-origin
Cross-Origin-Embedder-Policy: require-corp
2 changes: 2 additions & 0 deletions test/fixtures/wpt/encoding/textdecoder-copy.any.js.headers
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cross-Origin-Opener-Policy: same-origin
Cross-Origin-Embedder-Policy: require-corp
40 changes: 40 additions & 0 deletions test/fixtures/wpt/encoding/textdecoder-eof.any.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
test(() => {
// Truncated sequences
assert_equals(new TextDecoder().decode(new Uint8Array([0xF0])), "\uFFFD");
assert_equals(new TextDecoder().decode(new Uint8Array([0xF0, 0x9F])), "\uFFFD");
assert_equals(new TextDecoder().decode(new Uint8Array([0xF0, 0x9F, 0x92])), "\uFFFD");

// Errors near end-of-queue
assert_equals(new TextDecoder().decode(new Uint8Array([0xF0, 0x9F, 0x41])), "\uFFFDA");
assert_equals(new TextDecoder().decode(new Uint8Array([0xF0, 0x41, 0x42])), "\uFFFDAB");
assert_equals(new TextDecoder().decode(new Uint8Array([0xF0, 0x41, 0xF0])), "\uFFFDA\uFFFD");
assert_equals(new TextDecoder().decode(new Uint8Array([0xF0, 0x8F, 0x92])), "\uFFFD\uFFFD\uFFFD");
}, "TextDecoder end-of-queue handling");

test(() => {
const decoder = new TextDecoder();
decoder.decode(new Uint8Array([0xF0]), { stream: true });
assert_equals(decoder.decode(), "\uFFFD");

decoder.decode(new Uint8Array([0xF0]), { stream: true });
decoder.decode(new Uint8Array([0x9F]), { stream: true });
assert_equals(decoder.decode(), "\uFFFD");

decoder.decode(new Uint8Array([0xF0, 0x9F]), { stream: true });
assert_equals(decoder.decode(new Uint8Array([0x92])), "\uFFFD");

assert_equals(decoder.decode(new Uint8Array([0xF0, 0x9F]), { stream: true }), "");
assert_equals(decoder.decode(new Uint8Array([0x41]), { stream: true }), "\uFFFDA");
assert_equals(decoder.decode(), "");

assert_equals(decoder.decode(new Uint8Array([0xF0, 0x41, 0x42]), { stream: true }), "\uFFFDAB");
assert_equals(decoder.decode(), "");

assert_equals(decoder.decode(new Uint8Array([0xF0, 0x41, 0xF0]), { stream: true }), "\uFFFDA");
assert_equals(decoder.decode(), "\uFFFD");

assert_equals(decoder.decode(new Uint8Array([0xF0]), { stream: true }), "");
assert_equals(decoder.decode(new Uint8Array([0x8F]), { stream: true }), "\uFFFD\uFFFD");
assert_equals(decoder.decode(new Uint8Array([0x92]), { stream: true }), "\uFFFD");
assert_equals(decoder.decode(), "");
}, "TextDecoder end-of-queue handling using stream: true");
53 changes: 50 additions & 3 deletions test/fixtures/wpt/encoding/textdecoder-streaming.any.js
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,62 @@ var octets = {
var decoder = new TextDecoder(encoding);
for (var i = 0; i < encoded.length; i += len) {
var sub = [];
for (var j = i; j < encoded.length && j < i + len; ++j)
for (var j = i; j < encoded.length && j < i + len; ++j) {
sub.push(encoded[j]);
var uintArray = new Uint8Array(createBuffer(arrayBufferOrSharedArrayBuffer, sub.length));
uintArray.set(sub);
}
var uintArray = new Uint8Array(createBuffer(arrayBufferOrSharedArrayBuffer, sub.length));
uintArray.set(sub);
out += decoder.decode(uintArray, {stream: true});
}
out += decoder.decode();
assert_equals(out, string);
}, 'Streaming decode: ' + encoding + ', ' + len + ' byte window (' + arrayBufferOrSharedArrayBuffer + ')');
}
});

test(() => {
function bytes(byteArray) {
const view = new Uint8Array(createBuffer(arrayBufferOrSharedArrayBuffer, byteArray.length));
view.set(byteArray);
return view;
}

const decoder = new TextDecoder();

assert_equals(decoder.decode(bytes([0xC1]), {stream: true}), "\uFFFD");
assert_equals(decoder.decode(), "");

assert_equals(decoder.decode(bytes([0xF5]), {stream: true}), "\uFFFD");
assert_equals(decoder.decode(), "");

assert_equals(decoder.decode(bytes([0xE0, 0x41]), {stream: true}), "\uFFFDA");
assert_equals(decoder.decode(bytes([0x42])), "B");

assert_equals(decoder.decode(bytes([0xE0, 0x80]), {stream: true}), "\uFFFD\uFFFD");
assert_equals(decoder.decode(bytes([0x80])), "\uFFFD");

assert_equals(decoder.decode(bytes([0xED, 0xA0]), {stream: true}), "\uFFFD\uFFFD");
assert_equals(decoder.decode(bytes([0x80])), "\uFFFD");

assert_equals(decoder.decode(bytes([0xF0, 0x41]), {stream: true}), "\uFFFDA");
assert_equals(decoder.decode(bytes([0x42]), {stream: true}), "B");
assert_equals(decoder.decode(bytes([0x43])), "C");

assert_equals(decoder.decode(bytes([0xF0, 0x80]), {stream: true}), "\uFFFD\uFFFD");
assert_equals(decoder.decode(bytes([0x80]), {stream: true}), "\uFFFD");
assert_equals(decoder.decode(bytes([0x80])), "\uFFFD");

assert_equals(decoder.decode(bytes([0xF4, 0xA0]), {stream: true}), "\uFFFD\uFFFD");
assert_equals(decoder.decode(bytes([0x80]), {stream: true}), "\uFFFD");
assert_equals(decoder.decode(bytes([0x80])), "\uFFFD");

assert_equals(decoder.decode(bytes([0xF0, 0x90, 0x41]), {stream: true}), "\uFFFDA");
assert_equals(decoder.decode(bytes([0x42])), "B");

// 4-byte UTF-8 sequences always correspond to non-BMP characters. Here
// we make sure that, although the first 3 bytes are enough to emit the
// lead surrogate, it only gets emitted when the fourth byte is read.
assert_equals(decoder.decode(bytes([0xF0, 0x9F, 0x92]), {stream: true}), "");
assert_equals(decoder.decode(bytes([0xA9])), "\u{1F4A9}");
}, `Streaming decode: UTF-8 chunk tests (${arrayBufferOrSharedArrayBuffer})`);
})
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cross-Origin-Opener-Policy: same-origin
Cross-Origin-Embedder-Policy: require-corp
2 changes: 1 addition & 1 deletion test/fixtures/wpt/versions.json
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
"path": "dom/events"
},
"encoding": {
"commit": "35f70910d3753c8b650fdfd4c716caedfefe88c9",
"commit": "c1b24fce6e625c1b79124a58f27bf9adce02d5d7",
"path": "encoding"
},
"FileAPI": {
Expand Down
15 changes: 10 additions & 5 deletions test/wpt/status/encoding.json
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,7 @@
"requires": ["small-icu"]
},
"textdecoder-streaming.any.js": {
"requires": ["small-icu"],
"fail": "WebAssembly.Memory does not support shared:true"
"requires": ["small-icu"]
},
"textdecoder-utf16-surrogates.any.js": {
"requires": ["small-icu"]
Expand Down Expand Up @@ -53,10 +52,10 @@
"fail": "No implementation of TextDecoderStream and TextEncoderStream"
},
"encodeInto.any.js": {
"fail": "WebAssembly.Memory does not support shared:true"
"requires": ["small-icu"]
},
"textdecoder-copy.any.js": {
"fail": "WebAssembly.Memory does not support shared:true"
"requires": ["small-icu"]
},
"legacy-mb-schinese/gbk/gbk-decoder.any.js": {
"requires": ["full-icu"],
Expand All @@ -67,6 +66,12 @@
"skip": "The gb18030 encoding is not supported"
},
"textdecoder-arguments.any.js": {
"fail": "Does not support flushing an incomplete sequence"
"requires": ["small-icu"]
},
"single-byte-decoder.window.js": {
"skip": "location is not defined"
},
"textdecoder-eof.any.js": {
"requires": ["small-icu"]
}
}

0 comments on commit 198cf59

Please sign in to comment.