Add latency measurements to mock server tests

danieljbruce · danieljbruce · commit d3d05a2f03b3 · 2024-05-23T16:18:40.000-04:00
This commit contains a tool useful for printing latency results.
diff --git a/latency-results.json b/latency-results.json
@@ -0,0 +1 @@
+[{"requestedOptionsWithTime":[{"time":1764,"rowRanges":[{}],"rowKeys":[]}],"name":"simple read","max_retries":3,"request_options":[{"rowKeys":[],"rowRanges":[{}]}],"responses":[],"row_keys_read":[["a","b","c"]]},{"requestedOptionsWithTime":[{"time":3,"rowRanges":[{}],"rowKeys":[]},{"time":19,"rowRanges":[{"startKeyOpen":"b"}],"rowKeys":[]}],"name":"retries a failed read","max_retries":3,"request_options":[{"rowKeys":[],"rowRanges":[{}]},{"rowKeys":[],"rowRanges":[{"startKeyOpen":"b"}]}],"responses":[],"row_keys_read":[["a","b"],["c"]]},{"requestedOptionsWithTime":[{"time":2,"rowRanges":[{}],"rowKeys":[]},{"time":10,"rowRanges":[{}],"rowKeys":[]},{"time":19,"rowRanges":[{}],"rowKeys":[]},{"time":29,"rowRanges":[{}],"rowKeys":[]}],"name":"fails after all available retries","max_retries":3,"request_options":[{"rowKeys":[],"rowRanges":[{}]},{"rowKeys":[],"rowRanges":[{}]},{"rowKeys":[],"rowRanges":[{}]},{"rowKeys":[],"rowRanges":[{}]}],"responses":[],"row_keys_read":[[],[],[],[]],"error":4},{"requestedOptionsWithTime":[{"time":2,"rowRanges":[{}],"rowKeys":[]},{"time":10,"rowRanges":[{"startKeyOpen":"a"}],"rowKeys":[]},{"time":20,"rowRanges":[{"startKeyOpen":"a"}],"rowKeys":[]},{"time":28,"rowRanges":[{"startKeyOpen":"a"}],"rowKeys":[]},{"time":37,"rowRanges":[{"startKeyOpen":"a"}],"rowKeys":[]},{"time":46,"rowRanges":[{"startKeyOpen":"b"}],"rowKeys":[]},{"time":55,"rowRanges":[{"startKeyOpen":"b"}],"rowKeys":[]},{"time":63,"rowRanges":[{"startKeyOpen":"b"}],"rowKeys":[]}],"name":"resets the retry counter after a successful read","max_retries":4,"request_options":[{"rowKeys":[],"rowRanges":[{}]},{"rowKeys":[],"rowRanges":[{"startKeyOpen":"a"}]},{"rowKeys":[],"rowRanges":[{"startKeyOpen":"a"}]},{"rowKeys":[],"rowRanges":[{"startKeyOpen":"a"}]},{"rowKeys":[],"rowRanges":[{"startKeyOpen":"a"}]},{"rowKeys":[],"rowRanges":[{"startKeyOpen":"b"}]},{"rowKeys":[],"rowRanges":[{"startKeyOpen":"b"}]},{"rowKeys":[],"rowRanges":[{"startKeyOpen":"b"}]}],"responses":[],"row_keys_read":[["a"],[],[],[],["b"],[],[],["c"]]},{"requestedOptionsWithTime":[{"time":2,"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"z"}],"rowKeys":[]},{"time":9,"rowRanges":[{"startKeyOpen":"b","endKeyClosed":"z"}],"rowKeys":[]}],"name":"moves the start point of a range being consumed","max_retries":3,"createReadStream_options":{"ranges":[{"start":{"value":"b","inclusive":false},"end":"z"}]},"request_options":[{"rowKeys":[],"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"z"}]},{"rowKeys":[],"rowRanges":[{"startKeyOpen":"b","endKeyClosed":"z"}]}],"responses":[],"row_keys_read":[["a","b"],["c"]]},{"requestedOptionsWithTime":[{"time":2,"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"c"},{"startKeyClosed":"x","endKeyClosed":"z"}],"rowKeys":[]},{"time":10,"rowRanges":[{"startKeyClosed":"x","endKeyClosed":"z"}],"rowKeys":[]}],"name":"removes ranges already consumed","max_retries":3,"createReadStream_options":{"ranges":[{"start":"x","end":"z"}]},"request_options":[{"rowKeys":[],"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"c"},{"startKeyClosed":"x","endKeyClosed":"z"}]},{"rowKeys":[],"rowRanges":[{"startKeyClosed":"x","endKeyClosed":"z"}]}],"responses":[],"row_keys_read":[["a","b","c"],["x"]]},{"requestedOptionsWithTime":[{"time":2,"rowRanges":[],"rowKeys":["a","b","x"]},{"time":9,"rowRanges":[],"rowKeys":["x"]}],"name":"removes keys already read","max_retries":3,"createReadStream_options":{"keys":["a","b","x"]},"request_options":[{"rowKeys":["a","b","x"],"rowRanges":[]},{"rowKeys":["x"],"rowRanges":[]}],"responses":[],"row_keys_read":[["a","b","c"],["x"]]},{"requestedOptionsWithTime":[{"time":3,"rowRanges":[{}],"rowKeys":[],"rowsLimit":10},{"time":10,"rowRanges":[{"startKeyOpen":"b"}],"rowKeys":[],"rowsLimit":8}],"name":"adjust the limit based on the number of rows read","max_retries":3,"createReadStream_options":{"limit":10},"request_options":[{"rowKeys":[],"rowRanges":[{}],"rowsLimit":10},{"rowsLimit":8,"rowKeys":[],"rowRanges":[{"startKeyOpen":"b"}]}],"responses":[],"row_keys_read":[["a","b"],["x"]]},{"requestedOptionsWithTime":[{"time":2,"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"c"},{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}],"rowKeys":["a","b","c","p","q","r","s","x","y","z"],"rowsLimit":10},{"time":9,"rowRanges":[{"startKeyOpen":"a","endKeyClosed":"c"},{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}],"rowKeys":["b","c","p","q","r","s","x","y","z"],"rowsLimit":9},{"time":18,"rowRanges":[{"startKeyOpen":"b","endKeyClosed":"c"},{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}],"rowKeys":["c","p","q","r","s","x","y","z"],"rowsLimit":8},{"time":26,"rowRanges":[{"startKeyOpen":"b","endKeyClosed":"c"},{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}],"rowKeys":["c","p","q","r","s","x","y","z"],"rowsLimit":8},{"time":43,"rowRanges":[{"startKeyOpen":"b","endKeyClosed":"c"},{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}],"rowKeys":["c","p","q","r","s","x","y","z"],"rowsLimit":8},{"time":50,"rowRanges":[{"startKeyOpen":"b","endKeyClosed":"c"},{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}],"rowKeys":["c","p","q","r","s","x","y","z"],"rowsLimit":8},{"time":58,"rowRanges":[{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}],"rowKeys":["p","q","r","s","x","y","z"],"rowsLimit":7},{"time":67,"rowRanges":[{"startKeyOpen":"x","endKeyClosed":"z"}],"rowKeys":["y","z"],"rowsLimit":2},{"time":76,"rowRanges":[{"startKeyOpen":"y","endKeyClosed":"z"}],"rowKeys":["z"],"rowsLimit":1}],"name":"does the previous 5 things in one giant test case","max_retries":4,"createReadStream_options":{"limit":10,"ranges":[{"start":{"value":"y","inclusive":false},"end":"z"}],"keys":["a","b","c","p","q","r","s","x","y","z"]},"request_options":[{"rowKeys":["a","b","c","p","q","r","s","x","y","z"],"rowsLimit":10,"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"c"},{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}]},{"rowKeys":["b","c","p","q","r","s","x","y","z"],"rowsLimit":9,"rowRanges":[{"startKeyOpen":"a","endKeyClosed":"c"},{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}]},{"rowKeys":["c","p","q","r","s","x","y","z"],"rowsLimit":8,"rowRanges":[{"startKeyOpen":"b","endKeyClosed":"c"},{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}]},{"rowKeys":["c","p","q","r","s","x","y","z"],"rowsLimit":8,"rowRanges":[{"startKeyOpen":"b","endKeyClosed":"c"},{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}]},{"rowKeys":["c","p","q","r","s","x","y","z"],"rowsLimit":8,"rowRanges":[{"startKeyOpen":"b","endKeyClosed":"c"},{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}]},{"rowKeys":["c","p","q","r","s","x","y","z"],"rowsLimit":8,"rowRanges":[{"startKeyOpen":"b","endKeyClosed":"c"},{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}]},{"rowKeys":["p","q","r","s","x","y","z"],"rowsLimit":7,"rowRanges":[{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}]},{"rowKeys":["y","z"],"rowsLimit":2,"rowRanges":[{"startKeyOpen":"x","endKeyClosed":"z"}]},{"rowKeys":["z"],"rowsLimit":1,"rowRanges":[{"startKeyOpen":"y","endKeyClosed":"z"}]}],"responses":[],"row_keys_read":[["a"],["b"],[],[],[],["c"],["p","q","r","s","x"],["y"],["z"]]},{"requestedOptionsWithTime":[{"time":2,"rowRanges":[{}],"rowKeys":[]},{"time":8,"rowRanges":[{}],"rowKeys":[]}],"name":"should do a retry the stream is interrupted","createReadStream_options":{},"request_options":[{"rowKeys":[],"rowRanges":[{}]},{"rowKeys":[],"rowRanges":[{}]}],"responses":[],"row_keys_read":[[],["z"]]},{"requestedOptionsWithTime":[{"time":1,"rowRanges":[{}],"rowKeys":[]}],"name":"should not retry CANCELLED errors","createReadStream_options":{},"request_options":[{"rowKeys":[],"rowRanges":[{}]}],"responses":[],"row_keys_read":[[]]},{"requestedOptionsWithTime":[{"time":2,"rowRanges":[{}],"rowKeys":[]},{"time":8,"rowRanges":[{"startKeyOpen":"a"}],"rowKeys":[]}],"name":"should have a range which starts after the last read key","createReadStream_options":{},"request_options":[{"rowKeys":[],"rowRanges":[{}]},{"rowKeys":[],"rowRanges":[{"startKeyOpen":"a"}]}],"responses":[],"row_keys_read":[["a"],["z"]]},{"requestedOptionsWithTime":[{"time":2,"rowRanges":[{"startKeyClosed":"a"}],"rowKeys":[]},{"time":8,"rowRanges":[{"startKeyOpen":"a"}],"rowKeys":[]}],"name":"should move the active range start to after the last read key","createReadStream_options":{"ranges":[{"start":{"value":"a","inclusive":false}}]},"request_options":[{"rowKeys":[],"rowRanges":[{"startKeyClosed":"a"}]},{"rowKeys":[],"rowRanges":[{"startKeyOpen":"a"}]}],"responses":[],"row_keys_read":[["a"],["z"]]},{"requestedOptionsWithTime":[{"time":2,"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"b"},{"startKeyClosed":"c"}],"rowKeys":[]},{"time":8,"rowRanges":[{"startKeyClosed":"c"}],"rowKeys":[]}],"name":"should remove ranges which were already read","createReadStream_options":{"ranges":[{"start":"c"}]},"request_options":[{"rowKeys":[],"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"b"},{"startKeyClosed":"c"}]},{"rowKeys":[],"rowRanges":[{"startKeyClosed":"c"}]}],"responses":[],"row_keys_read":[["a","b"],["c"]]},{"requestedOptionsWithTime":[{"time":2,"rowRanges":[],"rowKeys":["a","b"]},{"time":16,"rowRanges":[],"rowKeys":["b"]}],"name":"should remove the keys which were already read","createReadStream_options":{"keys":["a","b"]},"request_options":[{"rowKeys":["a","b"],"rowRanges":[]},{"rowKeys":["b"],"rowRanges":[]}],"responses":[],"row_keys_read":[["a"],["c"]]},{"requestedOptionsWithTime":[{"time":1,"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"c"}],"rowKeys":[],"rowsLimit":2}],"name":"should not retry if limit is reached","createReadStream_options":{"ranges":[{"start":{"value":"b","inclusive":false},"end":"c"}],"limit":2},"request_options":[{"rowKeys":[],"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"c"}],"rowsLimit":2}],"responses":[],"row_keys_read":[["a","b"]],"error":4},{"requestedOptionsWithTime":[{"time":1,"rowRanges":[],"rowKeys":["a"]}],"name":"should not retry if all the keys are read","createReadStream_options":{"keys":["a"]},"request_options":[{"rowKeys":["a"],"rowRanges":[]}],"responses":[],"row_keys_read":[["a"]],"error":4},{"requestedOptionsWithTime":[{"time":2,"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"c"}],"rowKeys":[]}],"name":"should not retry if all the ranges are read","createReadStream_options":{"ranges":[]},"request_options":[{"rowKeys":[],"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"c"}]}],"responses":[],"row_keys_read":[["c"]],"error":4},{"requestedOptionsWithTime":[{"time":1,"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"b"}],"rowKeys":["c","d"]}],"name":"shouldn not retry with keys and ranges that are read","createReadStream_options":{"ranges":[],"keys":["c","d"]},"request_options":[{"rowKeys":["c","d"],"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"b"}]}],"responses":[],"row_keys_read":[["a1","d"]],"error":4},{"requestedOptionsWithTime":[{"time":1,"rowRanges":[],"rowKeys":["a"]},{"time":9,"rowRanges":[],"rowKeys":["a"]}],"name":"should retry received rst stream errors","createReadStream_options":{"keys":["a"]},"request_options":[{"rowKeys":["a"],"rowRanges":[]},{"rowKeys":["a"],"rowRanges":[]}],"responses":[],"row_keys_read":[[],["a"]]}]
diff --git a/system-test/read-rows.ts b/system-test/read-rows.ts
@@ -26,6 +26,7 @@ import {BigtableClientMockService} from '../src/util/mock-servers/service-implem
 import {ServerWritableStream} from '@grpc/grpc-js';
 
 const {grpc} = new GrpcClient();
+const fs = require('fs');
 
 function rowResponseFromServer(rowKey: string) {
   return {
@@ -129,6 +130,7 @@ describe('Bigtable/Table', () => {
   });
 
   describe.only('createReadStream using mock server', () => {
+    const timeResults: any[] = [];
     let server: MockServer;
     let service: MockService;
     let bigtable = new Bigtable();
@@ -147,6 +149,11 @@ describe('Bigtable/Table', () => {
 
     after(async () => {
       server.shutdown(() => {});
+      fs.writeFileSync(
+        './latency-results.json',
+        JSON.stringify(timeResults),
+        'utf8'
+      );
     });
 
     tests.forEach(test => {
@@ -159,7 +166,15 @@ describe('Bigtable/Table', () => {
         const rowKeysRead: string[][] = [];
         let endCalled = false;
         let error: ServiceError | null = null;
+        const testTime = Date.now();
+        const requestedOptionsWithTime: any[] = [];
         function checkResults() {
+          timeResults.push(
+            Object.assign(
+              {requestedOptionsWithTime: requestedOptionsWithTime},
+              test
+            )
+          );
           if (test.error) {
             assert(!endCalled, ".on('end') should not have been invoked");
             assert.strictEqual(error!.code, test.error);
@@ -189,6 +204,12 @@ describe('Bigtable/Table', () => {
             assert(response);
             rowKeysRead.push([]);
             requestedOptions.push(getRequestOptions(stream.request));
+            requestedOptionsWithTime.push(
+              Object.assign(
+                {time: Date.now() - testTime},
+                getRequestOptions(stream.request)
+              )
+            );
             if (response.row_keys) {
               stream.write({
                 chunks: response.row_keys.map(rowResponseFromServer),

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+[{"requestedOptionsWithTime":[{"time":1764,"rowRanges":[{}],"rowKeys":[]}],"name":"simple read","max_retries":3,"request_options":[{"rowKeys":[],"rowRanges":[{}]}],"responses":[],"row_keys_read":[["a","b","c"]]},{"requestedOptionsWithTime":[{"time":3,"rowRanges":[{}],"rowKeys":[]},{"time":19,"rowRanges":[{"startKeyOpen":"b"}],"rowKeys":[]}],"name":"retries a failed read","max_retries":3,"request_options":[{"rowKeys":[],"rowRanges":[{}]},{"rowKeys":[],"rowRanges":[{"startKeyOpen":"b"}]}],"responses":[],"row_keys_read":[["a","b"],["c"]]},{"requestedOptionsWithTime":[{"time":2,"rowRanges":[{}],"rowKeys":[]},{"time":10,"rowRanges":[{}],"rowKeys":[]},{"time":19,"rowRanges":[{}],"rowKeys":[]},{"time":29,"rowRanges":[{}],"rowKeys":[]}],"name":"fails after all available retries","max_retries":3,"request_options":[{"rowKeys":[],"rowRanges":[{}]},{"rowKeys":[],"rowRanges":[{}]},{"rowKeys":[],"rowRanges":[{}]},{"rowKeys":[],"rowRanges":[{}]}],"responses":[],"row_keys_read":[[],[],[],[]],"error":4},{"requestedOptionsWithTime":[{"time":2,"rowRanges":[{}],"rowKeys":[]},{"time":10,"rowRanges":[{"startKeyOpen":"a"}],"rowKeys":[]},{"time":20,"rowRanges":[{"startKeyOpen":"a"}],"rowKeys":[]},{"time":28,"rowRanges":[{"startKeyOpen":"a"}],"rowKeys":[]},{"time":37,"rowRanges":[{"startKeyOpen":"a"}],"rowKeys":[]},{"time":46,"rowRanges":[{"startKeyOpen":"b"}],"rowKeys":[]},{"time":55,"rowRanges":[{"startKeyOpen":"b"}],"rowKeys":[]},{"time":63,"rowRanges":[{"startKeyOpen":"b"}],"rowKeys":[]}],"name":"resets the retry counter after a successful read","max_retries":4,"request_options":[{"rowKeys":[],"rowRanges":[{}]},{"rowKeys":[],"rowRanges":[{"startKeyOpen":"a"}]},{"rowKeys":[],"rowRanges":[{"startKeyOpen":"a"}]},{"rowKeys":[],"rowRanges":[{"startKeyOpen":"a"}]},{"rowKeys":[],"rowRanges":[{"startKeyOpen":"a"}]},{"rowKeys":[],"rowRanges":[{"startKeyOpen":"b"}]},{"rowKeys":[],"rowRanges":[{"startKeyOpen":"b"}]},{"rowKeys":[],"rowRanges":[{"startKeyOpen":"b"}]}],"responses":[],"row_keys_read":[["a"],[],[],[],["b"],[],[],["c"]]},{"requestedOptionsWithTime":[{"time":2,"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"z"}],"rowKeys":[]},{"time":9,"rowRanges":[{"startKeyOpen":"b","endKeyClosed":"z"}],"rowKeys":[]}],"name":"moves the start point of a range being consumed","max_retries":3,"createReadStream_options":{"ranges":[{"start":{"value":"b","inclusive":false},"end":"z"}]},"request_options":[{"rowKeys":[],"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"z"}]},{"rowKeys":[],"rowRanges":[{"startKeyOpen":"b","endKeyClosed":"z"}]}],"responses":[],"row_keys_read":[["a","b"],["c"]]},{"requestedOptionsWithTime":[{"time":2,"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"c"},{"startKeyClosed":"x","endKeyClosed":"z"}],"rowKeys":[]},{"time":10,"rowRanges":[{"startKeyClosed":"x","endKeyClosed":"z"}],"rowKeys":[]}],"name":"removes ranges already consumed","max_retries":3,"createReadStream_options":{"ranges":[{"start":"x","end":"z"}]},"request_options":[{"rowKeys":[],"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"c"},{"startKeyClosed":"x","endKeyClosed":"z"}]},{"rowKeys":[],"rowRanges":[{"startKeyClosed":"x","endKeyClosed":"z"}]}],"responses":[],"row_keys_read":[["a","b","c"],["x"]]},{"requestedOptionsWithTime":[{"time":2,"rowRanges":[],"rowKeys":["a","b","x"]},{"time":9,"rowRanges":[],"rowKeys":["x"]}],"name":"removes keys already read","max_retries":3,"createReadStream_options":{"keys":["a","b","x"]},"request_options":[{"rowKeys":["a","b","x"],"rowRanges":[]},{"rowKeys":["x"],"rowRanges":[]}],"responses":[],"row_keys_read":[["a","b","c"],["x"]]},{"requestedOptionsWithTime":[{"time":3,"rowRanges":[{}],"rowKeys":[],"rowsLimit":10},{"time":10,"rowRanges":[{"startKeyOpen":"b"}],"rowKeys":[],"rowsLimit":8}],"name":"adjust the limit based on the number of rows read","max_retries":3,"createReadStream_options":{"limit":10},"request_options":[{"rowKeys":[],"rowRanges":[{}],"rowsLimit":10},{"rowsLimit":8,"rowKeys":[],"rowRanges":[{"startKeyOpen":"b"}]}],"responses":[],"row_keys_read":[["a","b"],["x"]]},{"requestedOptionsWithTime":[{"time":2,"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"c"},{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}],"rowKeys":["a","b","c","p","q","r","s","x","y","z"],"rowsLimit":10},{"time":9,"rowRanges":[{"startKeyOpen":"a","endKeyClosed":"c"},{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}],"rowKeys":["b","c","p","q","r","s","x","y","z"],"rowsLimit":9},{"time":18,"rowRanges":[{"startKeyOpen":"b","endKeyClosed":"c"},{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}],"rowKeys":["c","p","q","r","s","x","y","z"],"rowsLimit":8},{"time":26,"rowRanges":[{"startKeyOpen":"b","endKeyClosed":"c"},{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}],"rowKeys":["c","p","q","r","s","x","y","z"],"rowsLimit":8},{"time":43,"rowRanges":[{"startKeyOpen":"b","endKeyClosed":"c"},{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}],"rowKeys":["c","p","q","r","s","x","y","z"],"rowsLimit":8},{"time":50,"rowRanges":[{"startKeyOpen":"b","endKeyClosed":"c"},{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}],"rowKeys":["c","p","q","r","s","x","y","z"],"rowsLimit":8},{"time":58,"rowRanges":[{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}],"rowKeys":["p","q","r","s","x","y","z"],"rowsLimit":7},{"time":67,"rowRanges":[{"startKeyOpen":"x","endKeyClosed":"z"}],"rowKeys":["y","z"],"rowsLimit":2},{"time":76,"rowRanges":[{"startKeyOpen":"y","endKeyClosed":"z"}],"rowKeys":["z"],"rowsLimit":1}],"name":"does the previous 5 things in one giant test case","max_retries":4,"createReadStream_options":{"limit":10,"ranges":[{"start":{"value":"y","inclusive":false},"end":"z"}],"keys":["a","b","c","p","q","r","s","x","y","z"]},"request_options":[{"rowKeys":["a","b","c","p","q","r","s","x","y","z"],"rowsLimit":10,"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"c"},{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}]},{"rowKeys":["b","c","p","q","r","s","x","y","z"],"rowsLimit":9,"rowRanges":[{"startKeyOpen":"a","endKeyClosed":"c"},{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}]},{"rowKeys":["c","p","q","r","s","x","y","z"],"rowsLimit":8,"rowRanges":[{"startKeyOpen":"b","endKeyClosed":"c"},{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}]},{"rowKeys":["c","p","q","r","s","x","y","z"],"rowsLimit":8,"rowRanges":[{"startKeyOpen":"b","endKeyClosed":"c"},{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}]},{"rowKeys":["c","p","q","r","s","x","y","z"],"rowsLimit":8,"rowRanges":[{"startKeyOpen":"b","endKeyClosed":"c"},{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}]},{"rowKeys":["c","p","q","r","s","x","y","z"],"rowsLimit":8,"rowRanges":[{"startKeyOpen":"b","endKeyClosed":"c"},{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}]},{"rowKeys":["p","q","r","s","x","y","z"],"rowsLimit":7,"rowRanges":[{"startKeyClosed":"p","endKeyClosed":"s"},{"startKeyClosed":"x","endKeyClosed":"z"}]},{"rowKeys":["y","z"],"rowsLimit":2,"rowRanges":[{"startKeyOpen":"x","endKeyClosed":"z"}]},{"rowKeys":["z"],"rowsLimit":1,"rowRanges":[{"startKeyOpen":"y","endKeyClosed":"z"}]}],"responses":[],"row_keys_read":[["a"],["b"],[],[],[],["c"],["p","q","r","s","x"],["y"],["z"]]},{"requestedOptionsWithTime":[{"time":2,"rowRanges":[{}],"rowKeys":[]},{"time":8,"rowRanges":[{}],"rowKeys":[]}],"name":"should do a retry the stream is interrupted","createReadStream_options":{},"request_options":[{"rowKeys":[],"rowRanges":[{}]},{"rowKeys":[],"rowRanges":[{}]}],"responses":[],"row_keys_read":[[],["z"]]},{"requestedOptionsWithTime":[{"time":1,"rowRanges":[{}],"rowKeys":[]}],"name":"should not retry CANCELLED errors","createReadStream_options":{},"request_options":[{"rowKeys":[],"rowRanges":[{}]}],"responses":[],"row_keys_read":[[]]},{"requestedOptionsWithTime":[{"time":2,"rowRanges":[{}],"rowKeys":[]},{"time":8,"rowRanges":[{"startKeyOpen":"a"}],"rowKeys":[]}],"name":"should have a range which starts after the last read key","createReadStream_options":{},"request_options":[{"rowKeys":[],"rowRanges":[{}]},{"rowKeys":[],"rowRanges":[{"startKeyOpen":"a"}]}],"responses":[],"row_keys_read":[["a"],["z"]]},{"requestedOptionsWithTime":[{"time":2,"rowRanges":[{"startKeyClosed":"a"}],"rowKeys":[]},{"time":8,"rowRanges":[{"startKeyOpen":"a"}],"rowKeys":[]}],"name":"should move the active range start to after the last read key","createReadStream_options":{"ranges":[{"start":{"value":"a","inclusive":false}}]},"request_options":[{"rowKeys":[],"rowRanges":[{"startKeyClosed":"a"}]},{"rowKeys":[],"rowRanges":[{"startKeyOpen":"a"}]}],"responses":[],"row_keys_read":[["a"],["z"]]},{"requestedOptionsWithTime":[{"time":2,"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"b"},{"startKeyClosed":"c"}],"rowKeys":[]},{"time":8,"rowRanges":[{"startKeyClosed":"c"}],"rowKeys":[]}],"name":"should remove ranges which were already read","createReadStream_options":{"ranges":[{"start":"c"}]},"request_options":[{"rowKeys":[],"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"b"},{"startKeyClosed":"c"}]},{"rowKeys":[],"rowRanges":[{"startKeyClosed":"c"}]}],"responses":[],"row_keys_read":[["a","b"],["c"]]},{"requestedOptionsWithTime":[{"time":2,"rowRanges":[],"rowKeys":["a","b"]},{"time":16,"rowRanges":[],"rowKeys":["b"]}],"name":"should remove the keys which were already read","createReadStream_options":{"keys":["a","b"]},"request_options":[{"rowKeys":["a","b"],"rowRanges":[]},{"rowKeys":["b"],"rowRanges":[]}],"responses":[],"row_keys_read":[["a"],["c"]]},{"requestedOptionsWithTime":[{"time":1,"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"c"}],"rowKeys":[],"rowsLimit":2}],"name":"should not retry if limit is reached","createReadStream_options":{"ranges":[{"start":{"value":"b","inclusive":false},"end":"c"}],"limit":2},"request_options":[{"rowKeys":[],"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"c"}],"rowsLimit":2}],"responses":[],"row_keys_read":[["a","b"]],"error":4},{"requestedOptionsWithTime":[{"time":1,"rowRanges":[],"rowKeys":["a"]}],"name":"should not retry if all the keys are read","createReadStream_options":{"keys":["a"]},"request_options":[{"rowKeys":["a"],"rowRanges":[]}],"responses":[],"row_keys_read":[["a"]],"error":4},{"requestedOptionsWithTime":[{"time":2,"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"c"}],"rowKeys":[]}],"name":"should not retry if all the ranges are read","createReadStream_options":{"ranges":[]},"request_options":[{"rowKeys":[],"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"c"}]}],"responses":[],"row_keys_read":[["c"]],"error":4},{"requestedOptionsWithTime":[{"time":1,"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"b"}],"rowKeys":["c","d"]}],"name":"shouldn not retry with keys and ranges that are read","createReadStream_options":{"ranges":[],"keys":["c","d"]},"request_options":[{"rowKeys":["c","d"],"rowRanges":[{"startKeyClosed":"a","endKeyClosed":"b"}]}],"responses":[],"row_keys_read":[["a1","d"]],"error":4},{"requestedOptionsWithTime":[{"time":1,"rowRanges":[],"rowKeys":["a"]},{"time":9,"rowRanges":[],"rowKeys":["a"]}],"name":"should retry received rst stream errors","createReadStream_options":{"keys":["a"]},"request_options":[{"rowKeys":["a"],"rowRanges":[]},{"rowKeys":["a"],"rowRanges":[]}],"responses":[],"row_keys_read":[[],["a"]]}]