ARROW-1693: [JS] Expand JavaScript implementation, build system, fix integration tests

.travis.yml

@@ -87,7 +87,7 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_script_manylinux.sh
   - language: java
     os: linux
-    jdk: openjdk8
+    jdk: openjdk7
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh
   - language: java
@@ -103,23 +103,24 @@ matrix:
   - language: java
     os: linux
     env: ARROW_TEST_GROUP=integration
-    jdk: openjdk7
+    jdk: openjdk8
     before_script:
     - source $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
     - export CC="clang-4.0"
     - export CXX="clang++-4.0"
+    - nvm install node
     - $TRAVIS_BUILD_DIR/ci/travis_lint.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
     - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
     script:
     - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh
-  # TODO(wesm): Re-enable after issues in ARROW-1409 resolved
-  # - language: node_js
-  #   os: linux
-  #   node_js: node
-  #   before_script:
-  #   - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
-  #   script:
-  #   - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh
+  - language: node_js
+    os: linux
+    node_js: node
+    before_script:
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
+    script:
+    - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh
   - compiler: gcc
     language: cpp
     os: linux

LICENSE.txt

@@ -457,3 +457,98 @@ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
 ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the Boost project
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the mapbox/variant project, BSD 3-clause
+license
+
+Copyright (c) MapBox
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+- Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+- Redistributions in binary form must reproduce the above copyright notice, this
+  list of conditions and the following disclaimer in the documentation and/or
+  other materials provided with the distribution.
+- Neither the name "MapBox" nor the names of its contributors may be
+  used to endorse or promote products derived from this software without
+  specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the FlatBuffers project
+
+Copyright 2014 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the tslib project
+
+Copyright 2015 Microsoft Corporation. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

ci/travis_script_integration.sh

@@ -44,3 +44,22 @@ conda install -y pip numpy six
 python integration_test.py --debug
 
 popd
+
+pushd $ARROW_JS_DIR
+
+# lint and compile JS source
+npm run lint
+npm run build
+# create initial test data
+npm run test:createTestData
+# run once to write the snapshots
+npm test -- -t ts -u --integration
+# run again to test all builds against the snapshots
+npm test -- --integration
+# run tests against source to generate coverage data
+npm run test:coverage -- --integration
+# Uncomment to upload to coveralls
+# cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js;
+
+
+popd

ci/travis_script_js.sh

@@ -17,16 +17,15 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -e
+set -ex
 
-JS_DIR=${TRAVIS_BUILD_DIR}/js
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
-pushd $JS_DIR
+pushd $ARROW_JS_DIR
 
-npm run validate
-
-# Uncomment to use coveralls
-# npm run test:coverage
-# cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js;
+npm run lint
+npm run build
+# run the non-snapshot unit tests
+npm test
 
 popd

integration/integration_test.py

@@ -28,6 +28,7 @@
 import subprocess
 import tempfile
 import uuid
+import errno
 
 import numpy as np
 
@@ -198,9 +199,18 @@ def __init__(self, name, is_signed, bit_width, nullable=True,
         self.min_value = min_value
         self.max_value = max_value
 
-    @property
-    def numpy_type(self):
-        return ('int' if self.is_signed else 'uint') + str(self.bit_width)
+    def _get_generated_data_bounds(self):
+        signed_iinfo = np.iinfo('int' + str(self.bit_width))
+        if self.is_signed:
+            min_value, max_value = signed_iinfo.min, signed_iinfo.max
+        else:
+            # ARROW-1837 Remove this hack and restore full unsigned integer
+            # range
+            min_value, max_value = 0, signed_iinfo.max
+
+        lower_bound = max(min_value, self.min_value)
+        upper_bound = min(max_value, self.max_value)
+        return lower_bound, upper_bound
 
     def _get_type(self):
         return OrderedDict([
@@ -210,9 +220,7 @@ def _get_type(self):
         ])
 
     def generate_column(self, size, name=None):
-        iinfo = np.iinfo(self.numpy_type)
-        lower_bound = max(iinfo.min, self.min_value)
-        upper_bound = min(iinfo.max, self.max_value)
+        lower_bound, upper_bound = self._get_generated_data_bounds()
         return self.generate_range(size, lower_bound, upper_bound, name=name)
 
     def generate_range(self, size, lower, upper, name=None):
@@ -521,7 +529,7 @@ def get_json(self):
 class BinaryColumn(PrimitiveColumn):
 
     def _encode_value(self, x):
-        return frombytes(binascii.hexlify(x))
+        return frombytes(binascii.hexlify(x).upper())
 
     def _get_buffers(self):
         offset = 0
@@ -785,7 +793,7 @@ def _generate_file(name, fields, batch_sizes, dictionaries=None):
     return JsonFile(name, schema, batches, dictionaries)
 
 
-def generate_primitive_case(batch_sizes):
+def generate_primitive_case(batch_sizes, name='primitive'):
     types = ['bool', 'int8', 'int16', 'int32', 'int64',
              'uint8', 'uint16', 'uint32', 'uint64',
              'float32', 'float64', 'binary', 'utf8']
@@ -796,7 +804,7 @@ def generate_primitive_case(batch_sizes):
         fields.append(get_field(type_ + "_nullable", type_, True))
         fields.append(get_field(type_ + "_nonnullable", type_, False))
 
-    return _generate_file("primitive", fields, batch_sizes)
+    return _generate_file(name, fields, batch_sizes)
 
 
 def generate_decimal_case():
@@ -874,8 +882,8 @@ def _temp_path():
         return
 
     file_objs = [
-        generate_primitive_case([7, 10]),
-        generate_primitive_case([0, 0, 0]),
+        generate_primitive_case([17, 20], name='primitive'),
+        generate_primitive_case([0, 0, 0], name='primitive_zerolength'),
         generate_decimal_case(),
         generate_datetime_case(),
         generate_nested_case(),
@@ -1079,11 +1087,33 @@ def run_all_tests(debug=False):
     print('-- All tests passed!')
 
 
+def write_js_test_json(directory):
+    generate_nested_case().write(os.path.join(directory, 'nested.json'))
+    generate_decimal_case().write(os.path.join(directory, 'decimal.json'))
+    generate_datetime_case().write(os.path.join(directory, 'datetime.json'))
+    (generate_dictionary_case()
+     .write(os.path.join(directory, 'dictionary.json')))
+    (generate_primitive_case([7, 10])
+     .write(os.path.join(directory, 'primitive.json')))
+    (generate_primitive_case([0, 0, 0])
+     .write(os.path.join(directory, 'primitive-empty.json')))
+
+
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Arrow integration test CLI')
+    parser.add_argument('--write_generated_json', dest='generated_json_path',
+                        action='store', default=False,
+                        help='Generate test JSON')
     parser.add_argument('--debug', dest='debug', action='store_true',
                         default=False,
                         help='Run executables in debug mode as relevant')
-
     args = parser.parse_args()
-    run_all_tests(debug=args.debug)
+    if args.generated_json_path:
+        try:
+            os.makedirs(args.generated_json_path)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+        write_js_test_json(args.generated_json_path)
+    else:
+        run_all_tests(debug=args.debug)

js/.gitignore

@@ -83,3 +83,8 @@ package-lock.json
 # compilation targets
 dist
 targets
+
+# test data files
+test/data/
+# jest snapshots (too big)
+test/__snapshots__/