Rename Bytes -> TextEncodingNone

Author: guilhermeleobas

doc/omnisci/bytes.rst

@@ -1,8 +1,8 @@
 from .omnisci_array import *  # noqa: F401, F403
 from .omnisci_column import *  # noqa: F401, F403
-from .omnisci_bytes import *  # noqa: F401, F403
 from .omnisci_metatype import *  # noqa: F401, F403
-from .omnisci_text_encoding import *  # noqa: F401, F403
+from .omnisci_text_encoding_dict import *  # noqa: F401, F403
+from .omnisci_text_encoding_none import *  # noqa: F401, F403
 from .omnisci_pipeline import *  # noqa: F401, F403
 from .omnisci_column_list import *  # noqa: F401, F403
 from .omnisci_table_function_manager import *  # noqa: F401, F403

rbc/omnisci_backend/__init__.py

@@ -158,7 +158,7 @@ def define_pipelines(self):
         # namely the "nopython" pipeline
         pm = DefaultPassBuilder.define_nopython_pipeline(self.state)
         # Add the new pass to run after IRProcessing
-        pm.add_pass_after(AutoFreeBuffers, IRProcessing)
+        # pm.add_pass_after(AutoFreeBuffers, IRProcessing)
         pm.add_pass_after(CheckRaiseStmts, IRProcessing)
         pm.add_pass_after(DTypeComparison, ReconstructSSA)
         # finalize

rbc/omnisci_backend/omnisci_bytes.py

@@ -1,4 +1,4 @@
-'''Omnisci Bytes type that corresponds to Omnisci type TEXT ENCODED NONE.
+'''Omnisci TextEncodingDict type that corresponds to Omnisci type TEXT ENCODED DICT.
 '''
 
 __all__ = ['OmnisciTextEncodingDictType', 'TextEncodingDict']

rbc/omnisci_backend/omnisci_pipeline.py

@@ -0,0 +1,72 @@
+'''Omnisci TextEncodingNone type that corresponds to Omnisci type TEXT ENCODED NONE.
+'''
+
+__all__ = ['TextEncodingNonePointer', 'TextEncodingNone', 'OmnisciTextEncodingNoneType']
+
+from rbc import typesystem
+from .omnisci_buffer import (
+    BufferPointer, Buffer, OmnisciBufferType,
+    omnisci_buffer_constructor)
+from numba.core import types, extending
+
+
+class OmnisciTextEncodingNoneType(OmnisciBufferType):
+    """Omnisci TextEncodingNone type for RBC typesystem.
+    """
+
+    @classmethod
+    def preprocess_args(cls, args):
+        element_type = typesystem.Type.fromstring('char8')
+        return ((element_type,),)
+
+    @property
+    def buffer_extra_members(self):
+        return ('bool is_null',)
+
+    def match(self, other):
+        if type(self) is type(other):
+            return self[0] == other[0]
+        if other.is_pointer and other[0].is_char and other[0].bits == 8:
+            return 1
+        if other.is_string:
+            return 2
+
+
+TextEncodingNonePointer = BufferPointer
+
+
+class TextEncodingNone(Buffer):
+    '''Omnisci TextEncodingNone type that corresponds to Omnisci type TEXT ENCODED NONE.
+
+    Omnisci TextEncodingNone represents the following structure:
+
+    struct TextEncodingNone {
+        char* ptr;
+        size_t sz;  // when non-negative, TextEncodingNone has fixed width.
+    }
+
+
+    .. code-block:: python
+
+        from rbc.omnisci_backend import TextEncodingNone
+
+        @omnisci('TextEncodingNone(int32, int32)')
+        def make_abc(first, n):
+            r = TextEncodingNone(n)
+            for i in range(n):
+                r[i] = first + i
+            return r
+    '''
+    pass
+
+
+@extending.lower_builtin(TextEncodingNone, types.Integer)
+def omnisci_text_encoding_none_constructor(context, builder, sig, args):
+    return omnisci_buffer_constructor(context, builder, sig, args)
+
+
+@extending.type_callable(TextEncodingNone)
+def type_omnisci_text_encoding_none(context):
+    def typer(size):
+        return typesystem.Type.fromobject('TextEncodingNone').tonumba()
+    return typer

rbc/omnisci_backend/omnisci_text_encoding.py rbc/omnisci_backend/omnisci_text_encoding_dict.py

@@ -14,7 +14,7 @@
 from .thrift import Client as ThriftClient
 from . import omnisci_backend
 from .omnisci_backend import (
-    OmnisciArrayType, OmnisciBytesType, OmnisciTextEncodingDictType,
+    OmnisciArrayType, OmnisciTextEncodingNoneType, OmnisciTextEncodingDictType,
     OmnisciOutputColumnType, OmnisciColumnType,
     OmnisciCompilerPipeline, OmnisciCursorType,
     BufferMeta, OmnisciColumnListType, OmnisciTableFunctionManagerType)
@@ -389,7 +389,6 @@ def add(a, b):
     typesystem_aliases = dict(
         bool='bool8',
         Array='OmnisciArrayType',
-        Bytes='OmnisciBytesType<char8>',
         Cursor='OmnisciCursorType',
         Column='OmnisciColumnType',
         OutputColumn='OmnisciOutputColumnType',
@@ -399,6 +398,7 @@ def add(a, b):
         Constant='int32|sizer=Constant',
         PreFlight='int32|sizer=PreFlight',
         ColumnList='OmnisciColumnListType',
+        TextEncodingNone='OmnisciTextEncodingNoneType',
         TextEncodingDict='OmnisciTextEncodingDictType',
         TableFunctionManager='OmnisciTableFunctionManagerType<>',
         UDTF='int32|kind=UDTF'
@@ -849,7 +849,7 @@ def _get_ext_arguments_map(self):
             'GeoPolygon': typemap['TExtArgumentType'].get('GeoPolygon'),
             'GeoMultiPolygon': typemap['TExtArgumentType'].get(
                 'GeoMultiPolygon'),
-            'Bytes': typemap['TExtArgumentType'].get('TextEncodingNone'),
+            'TextEncodingNone': typemap['TExtArgumentType'].get('TextEncodingNone'),
             'TextEncodingDict': typemap['TExtArgumentType'].get('TextEncodingDict'),
             'ColumnList<bool>': typemap['TExtArgumentType'].get('ColumnListBool'),
             'ColumnList<int8_t>': typemap['TExtArgumentType'].get('ColumnListInt8'),
@@ -894,7 +894,8 @@ def _get_ext_arguments_map(self):
             ext_arguments_map['OmnisciOutputColumnListType<%s>' % ptr_type] \
                 = ext_arguments_map.get('ColumnList<%s>' % T)
 
-        ext_arguments_map['OmnisciBytesType<char8>'] = ext_arguments_map.get('Bytes')
+        ext_arguments_map['OmnisciTextEncodingNoneType<char8>'] = \
+            ext_arguments_map.get('TextEncodingNone')
 
         values = list(ext_arguments_map.values())
         for v, n in thrift.TExtArgumentType._VALUES_TO_NAMES.items():
@@ -1260,7 +1261,7 @@ def _register(self):
                     else:
                         skipped_names.append(udtf.name)
                 if self.debug:
-                    print(f'Skipping: {", ".join(skipped_names)}')
+                    print(f'Skipping: [{", ".join(skipped_names)}]')
         # Make sure that all registered functions have
         # implementations, otherwise, we will crash the server.
         for f in udtfs:
@@ -1397,8 +1398,8 @@ def format_type(self, typ: typesystem.Type):
         elif isinstance(typ, OmnisciCursorType):
             p = tuple(map(self.format_type, typ[0]))
             typ = typesystem.Type(('Cursor',) + p, **typ._params)
-        elif isinstance(typ, OmnisciBytesType):
-            typ = typ.copy().params(typename='Bytes')
+        elif isinstance(typ, OmnisciTextEncodingNoneType):
+            typ = typ.copy().params(typename='TextEncodingNone')
             use_typename = True
         elif isinstance(typ, OmnisciTextEncodingDictType):
             typ = typ.copy().params(typename='TextEncodingDict')
@@ -1443,7 +1444,7 @@ def remote_call(self, func, ftype: typesystem.Type, arguments: tuple, hold=False
 
             if isinstance(atype, (OmnisciColumnType, OmnisciColumnListType)):
                 args.append(f'CURSOR({a})')
-            elif isinstance(atype, OmnisciBytesType):
+            elif isinstance(atype, OmnisciTextEncodingNoneType):
                 if isinstance(a, bytes):
                     a = repr(a.decode())
                 elif isinstance(a, str):

rbc/omnisci_backend/omnisci_text_encoding_none.py

@@ -774,9 +774,9 @@ def test(s, caller=False):
     assert test('Column<int32> z') == 'Column<int32> z'
     assert test('Column<int32> | name=z') == 'Column<int32> z'
     assert test('Column<int32> x | name=z') == 'Column<int32> x | name=z'
-    assert test('Column<Bytes>') == 'Column<Bytes>'
+    assert test('Column<TextEncodingNone>') == 'Column<TextEncodingNone>'
     assert test('Column<Array<int32>>') == 'Column<Array<int32>>'
-    assert test('Column<Array<Bytes>>') == 'Column<Array<Bytes>>'
+    assert test('Column<Array<TextEncodingNone>>') == 'Column<Array<TextEncodingNone>>'
 
     assert test('OutputColumn<int32>') == 'OutputColumn<int32>'
     assert test('ColumnList<int32>') == 'ColumnList<int32>'
@@ -796,18 +796,18 @@ def test(s, caller=False):
     assert test('UDTF(Cursor<int32, float64>)') == 'UDTF(Cursor<Column<int32>, Column<float64>>)'
     assert test('UDTF(Cursor<int32 x>)') == 'UDTF(Cursor<Column<int32> x>)'
     assert test('UDTF(Cursor<int32 | name=x>)') == 'UDTF(Cursor<Column<int32> x>)'
-    assert test('UDTF(Cursor<Bytes x>)') == 'UDTF(Cursor<Column<Bytes> x>)'
+    assert test('UDTF(Cursor<TextEncodingNone x>)') == 'UDTF(Cursor<Column<TextEncodingNone> x>)'
 
     assert test('int32(int32)') == '(int32) -> int32'
     assert test('int32(int32 x)') == '(int32 x) -> int32'
     assert test('int32(Array<int32>)') == '(Array<int32>) -> int32'
     assert test('int32(int32[])') == '(Array<int32>) -> int32'
     assert test('int32(Array<int32> x)') == '(Array<int32> x) -> int32'
-    assert test('int32(Bytes)') == '(Bytes) -> int32'
-    assert test('int32(Bytes x)') == '(Bytes x) -> int32'
+    assert test('int32(TextEncodingNone)') == '(TextEncodingNone) -> int32'
+    assert test('int32(TextEncodingNone x)') == '(TextEncodingNone x) -> int32'
     assert test('int32(TextEncodingDict)') == '(TextEncodingDict) -> int32'
     assert test('int32(TextEncodingDict x)') == '(TextEncodingDict x) -> int32'
-    assert test('int32(Array<Bytes> x)') == '(Array<Bytes> x) -> int32'
+    assert test('int32(Array<TextEncodingNone> x)') == '(Array<TextEncodingNone> x) -> int32'
 
     def test2(s):
         return test(s, caller=True)
@@ -822,8 +822,8 @@ def test2(s):
     assert test2('UDTF(Constant, OutputColumn<int32>)') == '(void) -> (Column<int32>)'
     assert test2('UDTF(PreFlight, OutputColumn<int32>)') == '(void) -> (Column<int32>)'
     assert test2('UDTF(TableFunctionManager, OutputColumn<int32>)') == '(void) -> (Column<int32>)'
-    assert (test2('UDTF(RowMultiplier, OutputColumn<Array<Bytes>>)')
-            == '(RowMultiplier) -> (Column<Array<Bytes>>)')
+    assert (test2('UDTF(RowMultiplier, OutputColumn<Array<TextEncodingNone>>)')
+            == '(RowMultiplier) -> (Column<Array<TextEncodingNone>>)')
 
     assert test2('UDTF(Cursor<int32>)') == '(Cursor<Column<int32>>) -> void'
     assert test2('UDTF(Cursor<int32 x>)') == '(Cursor<Column<int32> x>) -> void'

rbc/omniscidb.py

@@ -1,7 +1,7 @@
 import numpy as np
 import pytest
 from rbc.externals.omniscidb import set_output_row_size
-from rbc.omnisci_backend import Bytes
+from rbc.omnisci_backend import TextEncodingNone
 from rbc.tests import omnisci_fixture, assert_equal
 
 
@@ -37,9 +37,9 @@ def aincr(x, dx, y):
             y[i] = x[i] + dx
         return size
 
-    @omnisci('Bytes(Bytes)')
+    @omnisci('TextEncodingNone(TextEncodingNone)')
     def myupper(s):
-        r = Bytes(len(s))
+        r = TextEncodingNone(len(s))
         for i in range(len(s)):
             c = s[i]
             if c >= 97 and c <= 122:
@@ -112,9 +112,9 @@ def test_remote_float64_evaluation(omnisci):
     assert_equal(arange(3, np.float64(1))['x'], np.arange(3, dtype=np.float64) + 1)
 
 
-def test_remote_bytes_evaluation(omnisci):
+def test_remote_TextEncodingNone_evaluation(omnisci):
     myupper = omnisci.get_caller('myupper')
-    assert str(myupper) == "myupper['Bytes(Bytes)']"
+    assert str(myupper) == "myupper['TextEncodingNone(TextEncodingNone)']"
     assert str(myupper("abc")) == "SELECT myupper('abc')"
     assert str(myupper("abc").execute()) == 'ABC'
     assert str(myupper(b"abc")) == "SELECT myupper('abc')"