Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

test(streams): improve DelimiterStream test cases #5070

Merged
merged 1 commit into from
Jun 18, 2024
Merged

test(streams): improve DelimiterStream test cases #5070

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion streams/delimiter_stream.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ export class DelimiterStream extends TransformStream<Uint8Array, Uint8Array> {
*/
constructor(
delimiter: Uint8Array,
options: DelimiterStreamOptions = { disposition: "discard" },
options: DelimiterStreamOptions = {},
) {
super({
transform: (chunk, controller) =>
Expand Down
82 changes: 76 additions & 6 deletions streams/delimiter_stream_test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ const DELIMITER_STREAM_INPUTS = [
"CRLFasd", // chunk starts with delimiter
].map((s) => new TextEncoder().encode(s));

Deno.test("DelimiterStream discard", async () => {
Deno.test("DelimiterStream handles { disposition: discard } correctly", async () => {
const crlf = new TextEncoder().encode("CRLF");
const delimStream = new DelimiterStream(crlf, { disposition: "discard" });
const outputs = [
Expand All @@ -39,7 +39,7 @@ Deno.test("DelimiterStream discard", async () => {
await testTransformStream(delimStream, DELIMITER_STREAM_INPUTS, outputs);
});

Deno.test("DelimiterStream suffix", async () => {
Deno.test("DelimiterStream handles { dispositioin: suffix } correctly", async () => {
const crlf = new TextEncoder().encode("CRLF");
const delimStream = new DelimiterStream(crlf, { disposition: "suffix" });
const outputs = [
Expand All @@ -59,7 +59,7 @@ Deno.test("DelimiterStream suffix", async () => {
await testTransformStream(delimStream, DELIMITER_STREAM_INPUTS, outputs);
});

Deno.test("DelimiterStream prefix", async () => {
Deno.test("DelimiterStream handles { dispositioin: prefix } correctly", async () => {
const crlf = new TextEncoder().encode("CRLF");
const delimStream = new DelimiterStream(crlf, { disposition: "prefix" });
const outputs = [
Expand All @@ -79,6 +79,25 @@ Deno.test("DelimiterStream prefix", async () => {
await testTransformStream(delimStream, DELIMITER_STREAM_INPUTS, outputs);
});

Deno.test("DelimiterStream handles { dispositioin: prefix } correctly when chunk starting with delimiter", async () => {
const crlf = new TextEncoder().encode("CRLF");
const delimStream = new DelimiterStream(crlf, { disposition: "prefix" });
const inputs = [
"CRLF123",
"4",
"5",
"CRLF67890CR",
"LF",
].map((s) => new TextEncoder().encode(s));
const outputs = [
"",
"CRLF12345",
"CRLF67890",
"CRLF",
].map((s) => new TextEncoder().encode(s));
await testTransformStream(delimStream, inputs, outputs);
});

const CHAR_DELIMITER_STREAM_INPUTS = [
"a", // more than one subsequent chunks with no delimiters
"b", // more than one subsequent chunks with no delimiters
Expand All @@ -95,7 +114,7 @@ const CHAR_DELIMITER_STREAM_INPUTS = [
"_asd", // chunk starts with delimiter
].map((s) => new TextEncoder().encode(s));

Deno.test("DelimiterStream char delimiter, discard", async () => {
Deno.test("DelimiterStream, with char delimiter, handles { disposition: discard } option correctly", async () => {
const delim = new TextEncoder().encode("_");
const delimStream = new DelimiterStream(delim, { disposition: "discard" });
const outputs = [
Expand All @@ -113,7 +132,7 @@ Deno.test("DelimiterStream char delimiter, discard", async () => {
await testTransformStream(delimStream, CHAR_DELIMITER_STREAM_INPUTS, outputs);
});

Deno.test("DelimiterStream char delimiter, suffix", async () => {
Deno.test("DelimiterStream, with char delimiter, handles { disposition: suffix } option correctly", async () => {
const delim = new TextEncoder().encode("_");
const delimStream = new DelimiterStream(delim, { disposition: "suffix" });
const outputs = [
Expand All @@ -131,7 +150,7 @@ Deno.test("DelimiterStream char delimiter, suffix", async () => {
await testTransformStream(delimStream, CHAR_DELIMITER_STREAM_INPUTS, outputs);
});

Deno.test("DelimiterStream char delimiter, prefix", async () => {
Deno.test("DelimiterStream, with char delimiter, handles { disposition: prefix } option correctly", async () => {
const delim = new TextEncoder().encode("_");
const delimStream = new DelimiterStream(delim, { disposition: "prefix" });
const outputs = [
Expand All @@ -149,6 +168,23 @@ Deno.test("DelimiterStream char delimiter, prefix", async () => {
await testTransformStream(delimStream, CHAR_DELIMITER_STREAM_INPUTS, outputs);
});

Deno.test("DelimiterStream, with char delimiter, handles { disposition: prefix } option correctly when chunk starting with delimiter", async () => {
const delim = new TextEncoder().encode("_");
const delimStream = new DelimiterStream(delim, { disposition: "prefix" });
const inputs = [
"_ab_ab",
"_cd_cd",
].map((s) => new TextEncoder().encode(s));
const outputs = [
"",
"_ab",
"_ab",
"_cd",
"_cd",
].map((s) => new TextEncoder().encode(s));
await testTransformStream(delimStream, inputs, outputs);
});

Deno.test("DelimiterStream regression 3609", async () => {
const delimStream = new DelimiterStream(new TextEncoder().encode(";"));
const inputs = [
Expand All @@ -169,3 +205,37 @@ Deno.test("DelimiterStream regression 3609", async () => {
].map((s) => new TextEncoder().encode(s));
await testTransformStream(delimStream, inputs, outputs);
});

Deno.test("DelimiterStream handles multiple chunks with no delimiters correctly", async () => {
// This tests flush implementation
const delimStream = new DelimiterStream(new TextEncoder().encode("|"));
const inputs = [
"a|b|c",
"d",
"e",
"f",
"|g",
"h",
"i",
].map((s) => new TextEncoder().encode(s));
const outputs = [
"a",
"b",
"cdef",
"ghi",
].map((s) => new TextEncoder().encode(s));
await testTransformStream(delimStream, inputs, outputs);
});

Deno.test("DelimiterStream handles delimiter AAB correctly when the input has AAABA", async () => {
// This tests flush implementation
const delimStream = new DelimiterStream(new TextEncoder().encode("AAB"));
const inputs = [
"AAABA",
].map((s) => new TextEncoder().encode(s));
const outputs = [
"A",
"A",
].map((s) => new TextEncoder().encode(s));
await testTransformStream(delimStream, inputs, outputs);
});