ARROW-4552: [JS] Add high-level Table and Column convenience methods

js/.vscode/launch.json

@@ -45,6 +45,9 @@
                 // "test/unit/table-tests.ts",
                 // "test/unit/generated-data-tests.ts",
 
+                // "test/unit/table/assign-tests.ts",
+                // "test/unit/table/serialize-tests.ts",
+
                 // "test/unit/vector/vector-tests.ts",
                 // "test/unit/vector/bool-vector-tests.ts",
                 // "test/unit/vector/date-vector-tests.ts",

js/src/column.ts

@@ -15,9 +15,11 @@
 // specific language governing permissions and limitations
 // under the License.
 
+import { Data } from './data';
 import { Field } from './schema';
-import { Vector } from './vector';
 import { DataType } from './type';
+import { Vector } from './vector';
+import { VectorCtorArgs, Vector as V } from './interfaces';
 import { Clonable, Sliceable, Applicative } from './vector';
 import { Chunked, SearchContinuation } from './vector/chunked';
 
@@ -34,6 +36,25 @@ export class Column<T extends DataType = any>
                Sliceable<Column<T>>,
                Applicative<T, Column<T>> {
 
+    public static new<T extends DataType>(field: string | Field<T>, ...chunks: (Vector<T> | Vector<T>[])[]): Column<T>;
+    public static new<T extends DataType>(field: string | Field<T>, data: Data<T>, ...args: VectorCtorArgs<V<T>>): Column<T>;
+    /** @nocollapse */
+    public static new<T extends DataType>(field: string | Field<T>, data: Data<T> | Vector<T> | Vector<T>[], ...rest: any[]): any {
+
+        const chunks = Chunked.flatten<T>(...(
+            Array.isArray(data) ? [...data, ...rest] :
+            data instanceof Vector ? [data, ...rest] :
+            [Vector.new(data, ...rest)]));
+
+        if (typeof field === 'string') {
+            const type = chunks[0].data.type;
+            field = new Field(field, type, chunks.some(({ nullCount }) => nullCount > 0));
+        } else if (!field.nullable && chunks.some(({ nullCount }) => nullCount > 0)) {
+            field = field.clone({ nullable: true });
+        }
+        return new Column(field, chunks);
+    }
+
     constructor(field: Field<T>, vectors: Vector<T>[] = [], offsets?: Uint32Array) {
         vectors = Chunked.flatten(...vectors);
         super(field.type, vectors, offsets);
@@ -48,6 +69,8 @@ export class Column<T extends DataType = any>
 
     public get field() { return this._field; }
     public get name() { return this._field.name; }
+    public get nullable() { return this._field.nullable; }
+    public get metadata() { return this._field.metadata; }
 
     public clone(chunks = this._chunks) {
         return new Column(this._field, chunks);

js/src/compute/dataframe.ts

@@ -83,11 +83,12 @@ export class DataFrame<T extends { [key: string]: DataType } = any> extends Tabl
 
 export class CountByResult<T extends DataType = any, TCount extends Int = Int> extends Table<{ values: T,  counts: TCount }> {
     constructor(values: Vector<T>, counts: V<TCount>) {
-        const schema = new Schema<{ values: T, counts: TCount }>([
+        type R = { values: T, counts: TCount };
+        const schema = new Schema<R>([
             new Field('values', values.type),
             new Field('counts', counts.type)
         ]);
-        super(new RecordBatch(schema, counts.length, [values, counts]));
+        super(new RecordBatch<R>(schema, counts.length, [values, counts]));
     }
     public toJSON(): Object {
         const values = this.getColumnAt(0)!;

js/src/data.ts

@@ -16,10 +16,11 @@
 // under the License.
 
 import { Vector } from './vector';
+import { truncateBitmap } from './util/bit';
 import { popcnt_bit_range } from './util/bit';
-import { toArrayBufferView } from './util/buffer';
 import { DataType, SparseUnion, DenseUnion } from './type';
 import { VectorType as BufferType, UnionMode, Type } from './enum';
+import { toArrayBufferView, toUint8Array, toInt32Array } from './util/buffer';
 import {
     Dictionary,
     Null, Int, Float,
@@ -139,6 +140,21 @@ export class Data<T extends DataType = DataType> {
             (!childData.length || this.valueOffsets) ? childData : this._sliceChildren(childData, childStride * offset, childStride * length));
     }
 
+    public _changeLengthAndBackfillNullBitmap(newLength: number): Data<T> {
+        const { length, nullCount } = this;
+        // start initialized with 0s (nulls), then fill from 0 to length with 1s (not null)
+        const bitmap = new Uint8Array(((newLength + 63) & ~63) >> 3).fill(255, 0, length >> 3);
+        // set all the bits in the last byte (up to bit `length - length % 8`) to 1 (not null)
+        bitmap[length >> 3] = (1 << (length - (length & ~7))) - 1;
+        // if we have a nullBitmap, truncate + slice and set it over the pre-filled 1s
+        if (nullCount > 0) {
+            bitmap.set(truncateBitmap(this.offset, length, this.nullBitmap), 0);
+        }
+        const buffers = this.buffers;
+        buffers[BufferType.VALIDITY] = bitmap;
+        return this.clone(this.type, 0, newLength, nullCount + (newLength - length), buffers);
+    }
+
     protected _sliceBuffers(offset: number, length: number, stride: number, typeId: T['TType']): Buffers<T> {
         let arr: any, { buffers } = this;
         // If typeIds exist, slice the typeIds buffer
@@ -159,85 +175,85 @@ export class Data<T extends DataType = DataType> {
     //
     /** @nocollapse */
     public static Null<T extends Null>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer) {
-        return new Data(type, offset, length, nullCount, [undefined, undefined, toArrayBufferView(Uint8Array, nullBitmap)]);
+        return new Data(type, offset, length, nullCount, [undefined, undefined, toUint8Array(nullBitmap)]);
     }
     /** @nocollapse */
     public static Int<T extends Int>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
-        return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toArrayBufferView(Uint8Array, nullBitmap)]);
+        return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]);
     }
     /** @nocollapse */
     public static Dictionary<T extends Dictionary>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
-        return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView<T['TArray']>(type.indices.ArrayType, data), toArrayBufferView(Uint8Array, nullBitmap)]);
+        return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView<T['TArray']>(type.indices.ArrayType, data), toUint8Array(nullBitmap)]);
     }
     /** @nocollapse */
     public static Float<T extends Float>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
-        return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toArrayBufferView(Uint8Array, nullBitmap)]);
+        return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]);
     }
     /** @nocollapse */
     public static Bool<T extends Bool>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
-        return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toArrayBufferView(Uint8Array, nullBitmap)]);
+        return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]);
     }
     /** @nocollapse */
     public static Decimal<T extends Decimal>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
-        return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toArrayBufferView(Uint8Array, nullBitmap)]);
+        return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]);
     }
     /** @nocollapse */
     public static Date<T extends Date_>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
-        return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toArrayBufferView(Uint8Array, nullBitmap)]);
+        return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]);
     }
     /** @nocollapse */
     public static Time<T extends Time>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
-        return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toArrayBufferView(Uint8Array, nullBitmap)]);
+        return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]);
     }
     /** @nocollapse */
     public static Timestamp<T extends Timestamp>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
-        return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toArrayBufferView(Uint8Array, nullBitmap)]);
+        return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]);
     }
     /** @nocollapse */
     public static Interval<T extends Interval>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
-        return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toArrayBufferView(Uint8Array, nullBitmap)]);
+        return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]);
     }
     /** @nocollapse */
     public static FixedSizeBinary<T extends FixedSizeBinary>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) {
-        return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toArrayBufferView(Uint8Array, nullBitmap)]);
+        return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]);
     }
     /** @nocollapse */
     public static Binary<T extends Binary>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, valueOffsets: ValueOffsetsBuffer, data: Uint8Array) {
-        return new Data(type, offset, length, nullCount, [toArrayBufferView(Int32Array, valueOffsets), toArrayBufferView(Uint8Array, data), toArrayBufferView(Uint8Array, nullBitmap)]);
+        return new Data(type, offset, length, nullCount, [toInt32Array(valueOffsets), toUint8Array(data), toUint8Array(nullBitmap)]);
     }
     /** @nocollapse */
     public static Utf8<T extends Utf8>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, valueOffsets: ValueOffsetsBuffer, data: Uint8Array) {
-        return new Data(type, offset, length, nullCount, [toArrayBufferView(Int32Array, valueOffsets), toArrayBufferView(Uint8Array, data), toArrayBufferView(Uint8Array, nullBitmap)]);
+        return new Data(type, offset, length, nullCount, [toInt32Array(valueOffsets), toUint8Array(data), toUint8Array(nullBitmap)]);
     }
     /** @nocollapse */
     public static List<T extends List>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, valueOffsets: ValueOffsetsBuffer, child: Data<T['valueType']> | Vector<T['valueType']>) {
-        return new Data(type, offset, length, nullCount, [toArrayBufferView(Int32Array, valueOffsets), undefined, toArrayBufferView(Uint8Array, nullBitmap)], [child]);
+        return new Data(type, offset, length, nullCount, [toInt32Array(valueOffsets), undefined, toUint8Array(nullBitmap)], [child]);
     }
     /** @nocollapse */
     public static FixedSizeList<T extends FixedSizeList>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, child: Data | Vector) {
-        return new Data(type, offset, length, nullCount, [undefined, undefined, toArrayBufferView(Uint8Array, nullBitmap)], [child]);
+        return new Data(type, offset, length, nullCount, [undefined, undefined, toUint8Array(nullBitmap)], [child]);
     }
     /** @nocollapse */
     public static Struct<T extends Struct>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, children: (Data | Vector)[]) {
-        return new Data(type, offset, length, nullCount, [undefined, undefined, toArrayBufferView(Uint8Array, nullBitmap)], children);
+        return new Data(type, offset, length, nullCount, [undefined, undefined, toUint8Array(nullBitmap)], children);
     }
     /** @nocollapse */
     public static Map<T extends Map_>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, children: (Data | Vector)[]) {
-        return new Data(type, offset, length, nullCount, [undefined, undefined, toArrayBufferView(Uint8Array, nullBitmap)], children);
+        return new Data(type, offset, length, nullCount, [undefined, undefined, toUint8Array(nullBitmap)], children);
     }
     public static Union<T extends SparseUnion>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, typeIds: TypeIdsBuffer, children: (Data | Vector)[]): Data<T>;
     public static Union<T extends DenseUnion>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, typeIds: TypeIdsBuffer, valueOffsets: ValueOffsetsBuffer, children: (Data | Vector)[]): Data<T>;
     /** @nocollapse */
     public static Union<T extends Union>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, typeIds: TypeIdsBuffer, valueOffsetsOrChildren: ValueOffsetsBuffer | (Data | Vector)[], children?: (Data | Vector)[]) {
         const buffers = <unknown> [
             undefined, undefined,
-            toArrayBufferView(Uint8Array, nullBitmap),
+            toUint8Array(nullBitmap),
             toArrayBufferView(type.ArrayType, typeIds)
         ] as Partial<Buffers<T>>;
         if (type.mode === UnionMode.Sparse) {
             return new Data(type, offset, length, nullCount, buffers, valueOffsetsOrChildren as (Data | Vector)[]);
         }
-        buffers[BufferType.OFFSET] = toArrayBufferView(Int32Array, <ValueOffsetsBuffer> valueOffsetsOrChildren);
+        buffers[BufferType.OFFSET] = toInt32Array(<ValueOffsetsBuffer> valueOffsetsOrChildren);
         return new Data(type, offset, length, nullCount, buffers, children);
     }
 }

js/src/interfaces.ts

@@ -76,7 +76,7 @@ export type VectorCtorType<
 export type Vector<T extends Type | DataType = any> =
     T extends Type          ? TypeToVector<T>     :
     T extends DataType      ? DataTypeToVector<T> :
-                              never
+                              vecs.BaseVector<any>
     ;
 
 /** @ignore */
@@ -192,7 +192,7 @@ type DataTypeToVector<T extends DataType = any> =
     ;
 
 /** @ignore */
-type TypeToDataType<T extends Type> =
+export type TypeToDataType<T extends Type> =
       T extends Type.Null                 ? type.Null
     : T extends Type.Bool                 ? type.Bool
     : T extends Type.Int                  ? type.Int

js/src/ipc/writer.ts

@@ -126,7 +126,7 @@ export class RecordBatchWriter<T extends { [key: string]: DataType } = any> exte
         this._dictionaryBlocks = [];
         this._recordBatchBlocks = [];
 
-        if (!schema || (schema !== this._schema)) {
+        if (!schema || !(schema.compareTo(this._schema))) {
             if (schema === null) {
                 this._position = 0;
                 this._schema = null;

js/src/recordbatch.ts

@@ -20,9 +20,9 @@ import { Table } from './table';
 import { Vector } from './vector';
 import { Schema, Field } from './schema';
 import { DataType, Struct } from './type';
-import { StructVector } from './vector/struct';
-import { Vector as VType } from './interfaces';
 import { Chunked } from './vector/chunked';
+import { StructVector } from './vector/struct';
+import { alignChunkLengths } from './util/recordbatch';
 import { Clonable, Sliceable, Applicative } from './vector';
 
 export interface RecordBatch<T extends { [key: string]: DataType } = any> {
@@ -38,28 +38,27 @@ export class RecordBatch<T extends { [key: string]: DataType } = any>
                Applicative<Struct<T>, Table<T>> {
 
     /** @nocollapse */
-    public static from<T extends { [key: string]: DataType } = any>(vectors: VType<T[keyof T]>[], names: (keyof T)[] = []) {
-        return new RecordBatch(
-            Schema.from(vectors, names),
-            vectors.reduce((len, vec) => Math.max(len, vec.length), 0),
-            vectors
-        );
+    public static from<T extends { [key: string]: DataType } = any>(chunks: (Data<T[keyof T]> | Vector<T[keyof T]>)[], names: (keyof T)[] = []) {
+        const schema = Schema.from(chunks, names);
+        const length = chunks.reduce((l, c) => Math.max(l, c.length), 0);
+        const children = chunks.map((x) => x instanceof Data ? x : x.data);
+        return new RecordBatch(schema, length, alignChunkLengths(schema, children, length));
     }
 
     protected _schema: Schema;
 
-    constructor(schema: Schema<T>, numRows: number, childData: (Data | Vector)[]);
+    constructor(schema: Schema<T>, length: number, children: (Data | Vector)[]);
     constructor(schema: Schema<T>, data: Data<Struct<T>>, children?: Vector[]);
     constructor(...args: any[]) {
-        let schema = args[0];
         let data: Data<Struct<T>>;
+        let schema = args[0] as Schema<T>;
         let children: Vector[] | undefined;
-        if (typeof args[1] === 'number') {
-            const fields = schema.fields as Field<T[keyof T]>[];
-            const [, numRows, childData] = args as [Schema<T>, number, Data[]];
-            data = Data.Struct(new Struct<T>(fields), 0, numRows, 0, null, childData);
+        if (args[1] instanceof Data) {
+            [, data, children] = (args as [any, Data<Struct<T>>, Vector<T[keyof T]>[]?]);
         } else {
-            [, data, children] = (args as [Schema<T>, Data<Struct<T>>, Vector[]?]);
+            const fields = schema.fields as Field<T[keyof T]>[];
+            const [, length, childData] = args as [any, number, Data<T[keyof T]>[]];
+            data = Data.Struct(new Struct<T>(fields), 0, length, 0, null, childData);
         }
         super(data, children);
         this._schema = schema;
@@ -85,4 +84,9 @@ export class RecordBatch<T extends { [key: string]: DataType } = any>
         const structData = Data.Struct(new Struct(schema.fields), 0, this.length, 0, null, childData);
         return new RecordBatch<{ [P in K]: T[P] }>(schema, structData as Data<Struct<{ [P in K]: T[P] }>>);
     }
+    public selectAt<K extends T[keyof T] = any>(...columnIndices: number[]) {
+        const schema = this._schema.selectAt(...columnIndices);
+        const children = columnIndices.map((i) => this.data.childData[i]);
+        return new RecordBatch<{ [key: string]: K }>(schema, this.length, children);
+    }
 }