Add variable length vector support

src/DotNetBridge/NativeDataInterop.cs

@@ -77,6 +77,12 @@ private struct DataViewBlock
             // key types. Zero means unbounded, -1 means not a key type.
             [FieldOffset(0x20)]
             public int* keyCards;
+
+            // The number of values in each row of a column.
+            // A value count of 0 means that each row of the column
+            // is variable length.
+            [FieldOffset(0x28)]
+            public byte* valueCounts;
         }
 
         private struct ColumnMetadataInfo
@@ -113,6 +119,7 @@ private static unsafe void SendViewToNativeAsDataFrame(IChannel ch, EnvironmentB
             var keyCardList = new List<int>();
             var nameUtf8Bytes = new List<Byte>();
             var nameIndices = new List<int>();
+            var valueCounts = new List<byte>();
 
             var expandCols = new HashSet<int>();
             var allNames = new HashSet<string>();
@@ -129,11 +136,7 @@ private static unsafe void SendViewToNativeAsDataFrame(IChannel ch, EnvironmentB
                 var kind = itemType.GetRawKind();
                 int keyCard;
 
-                if (fullType.GetValueCount() == 0)
-                {
-                    throw ch.ExceptNotSupp("Column has variable length vector: " + 
-                        name + ". Not supported in python. Drop column before sending to Python");
-                }
+                byte valueCount = (fullType.GetValueCount() == 0) ? (byte)0 : (byte)1;
 
                 if (itemType is KeyDataViewType)
                 {
@@ -232,22 +235,26 @@ private static unsafe void SendViewToNativeAsDataFrame(IChannel ch, EnvironmentB
                 {
                     kindList.Add(kind);
                     keyCardList.Add(keyCard);
+                    valueCounts.Add(valueCount);
                 }
             }
 
             ch.Assert(allNames.Count == kindList.Count);
             ch.Assert(allNames.Count == keyCardList.Count);
             ch.Assert(allNames.Count == nameIndices.Count);
+            ch.Assert(allNames.Count == valueCounts.Count);
 
             var kinds = kindList.ToArray();
             var keyCards = keyCardList.ToArray();
             var nameBytes = nameUtf8Bytes.ToArray();
             var names = new byte*[allNames.Count];
+            var valueCountsBytes = valueCounts.ToArray();
 
             fixed (InternalDataKind* prgkind = kinds)
             fixed (byte* prgbNames = nameBytes)
             fixed (byte** prgname = names)
             fixed (int* prgkeyCard = keyCards)
+            fixed (byte* prgbValueCount = valueCountsBytes)
             {
                 for (int iid = 0; iid < names.Length; iid++)
                     names[iid] = prgbNames + nameIndices[iid];
@@ -258,6 +265,7 @@ private static unsafe void SendViewToNativeAsDataFrame(IChannel ch, EnvironmentB
                 block.names = (sbyte**)prgname;
                 block.kinds = prgkind;
                 block.keyCards = prgkeyCard;
+                block.valueCounts = prgbValueCount;
 
                 dataSink(penv, &block, out var setters, out var keyValueSetter);
 

src/NativeBridge/DataViewInterop.h

@@ -524,6 +524,10 @@ struct DataViewBlock
     // Column key type cardinalities. Only contains the values for the columns that have
     // key names.
     const int *keyCards;
+    // The number of values in each row of a column.
+    // A value count of 0 means that each row of the column
+    // is variable length.
+    const BYTE *valueCounts;
 };
 
 enum ML_PY_TYPE_MAP_ENUM {

src/NativeBridge/ManagedInterop.cpp

@@ -6,63 +6,16 @@
 #include "DataViewInterop.h"
 #include "ManagedInterop.h"
 
-inline void destroyManagerCObject(PyObject* obj) {
-    auto* b = static_cast<PythonObjectBase*>(PyCapsule_GetPointer(obj, NULL));
-    if (b) { delete b; }
-}
-
-#define SetDict2(cpptype, nptype); \
-		{\
-			PythonObject<cpptype>* col = dynamic_cast<PythonObject<cpptype>*>(column);\
-			auto shrd = col->GetData();\
-			auto* data = shrd->data();\
-			bp::handle<> h(::PyCapsule_New((void*)column, NULL, (PyCapsule_Destructor)&destroyManagerCObject));\
-			dict[_names[i]] = np::from_data(\
-				data,\
-				np::dtype::get_builtin<nptype>(),\
-				bp::make_tuple(shrd->size()),\
-				bp::make_tuple(sizeof(nptype)), bp::object(h));\
-		}
 
-#define SetDict1(type) SetDict2(type, type)
-
-#define SetDictAndKeys(type, i); \
-		{\
-			PythonObject<type>* col = dynamic_cast<PythonObject<type>*>(column);\
-			auto shrd = col->GetData();\
-			auto* data = shrd->data();\
-			bp::handle<> h(::PyCapsule_New((void*)column, NULL, (PyCapsule_Destructor)&destroyManagerCObject));\
-			np::ndarray npdata = np::from_data(\
-				data,\
-				np::dtype::get_builtin<type>(),\
-				bp::make_tuple(shrd->size()),\
-				bp::make_tuple(sizeof(float)), bp::object(h));\
-			if (keyNames == nullptr)\
-			{\
-				dict[_names[i]] = npdata;\
-			}\
-			else\
-			{\
-				dict[_names[i]] = bp::dict();\
-				dict[_names[i]]["..Data"] = npdata;\
-				auto shrd = keyNames->GetData();\
-				bp::list list;\
-				for (int j = 0; j < shrd->size(); j++)\
-				{\
-					bp::object obj;\
-					const std::string& value = shrd->at(j);\
-					if (!value.empty())\
-					{\
-						obj = bp::object(value);\
-					}\
-					list.append(obj);\
-				}\
-				dict[_names[i]]["..KeyValues"] = list;\
-			}\
-		}\
+#define AddToDict(type); \
+        {\
+            PythonObject<type>* col = dynamic_cast<PythonObject<type>*>(column);\
+            col->AddToDict(dict, _names[i], keyNames);\
+        }\
 
 #define STATIC
 
+
 EnvironmentBlock::~EnvironmentBlock()
 {
     // Everything (except data buffers) that we might have exposed to managed code,
@@ -115,7 +68,7 @@ void EnvironmentBlock::DataSinkCore(const DataViewBlock * pdata)
     for (int i = 0; i < pdata->ccol; i++)
     {
         BYTE kind = pdata->kinds[i];
-        _columns.push_back(PythonObjectBase::CreateObject(kind, pdata->crow, 1));
+        _columns.push_back(PythonObjectBase::CreateObject(kind, pdata->crow, pdata->valueCounts[i]));
 
         switch (kind)
         {
@@ -164,7 +117,7 @@ void EnvironmentBlock::DataSinkCore(const DataViewBlock * pdata)
 
         if (pdata->keyCards && (pdata->keyCards[i] >= 0))
         {
-            _vKeyValues.push_back(new PythonObject<std::string>(TX, pdata->keyCards[i], 1));
+            _vKeyValues.push_back(new PythonObjectSingle<std::string>(TX, pdata->keyCards[i], 1));
             _columnToKeyMap.push_back(numKeys++);
         }
         else
@@ -238,15 +191,15 @@ bp::dict EnvironmentBlock::GetData()
     for (size_t i = 0; i < _names.size(); i++)
     {
         PythonObjectBase* column = _columns[i];
-        PythonObject<std::string>* keyNames = nullptr;
+        const std::vector<std::string>* keyNames = nullptr;
         if (_columnToKeyMap[i] >= 0)
-            keyNames = _vKeyValues[_columnToKeyMap[i]];
+            keyNames = _vKeyValues[_columnToKeyMap[i]]->GetData();
 
         signed char kind = column->GetKind();
         switch (kind) {
         case -1:
         {
-            PythonObject<signed char>* col = dynamic_cast<PythonObject<signed char>*>(column);
+            PythonObjectSingle<signed char>* col = dynamic_cast<PythonObjectSingle<signed char>*>(column);
             auto shrd = col->GetData();
             bp::list list;
             for (size_t i = 0; i < shrd->size(); i++)
@@ -266,57 +219,42 @@ bp::dict EnvironmentBlock::GetData()
         }
         break;
         case BL:
-            SetDict2(signed char, bool);
+            AddToDict(signed char);
             break;
         case I1:
-            SetDictAndKeys(signed char, i);
+            AddToDict(signed char);
             break;
         case I2:
-            SetDictAndKeys(signed short, i);
+            AddToDict(signed short);
             break;
         case I4:
-            SetDictAndKeys(signed int, i);
+            AddToDict(signed int);
             break;
         case I8:
-            SetDict1(CxInt64);
+            AddToDict(CxInt64);
             break;
         case U1:
-            SetDict1(unsigned char);
+            AddToDict(unsigned char);
             break;
         case U2:
-            SetDict1(unsigned short);
+            AddToDict(unsigned short);
             break;
         case U4:
-            SetDict1(unsigned int);
+            AddToDict(unsigned int);
             break;
         case U8:
-            SetDict1(CxUInt64);
+            AddToDict(CxUInt64);
             break;
         case R4:
-            SetDict1(float);
+            AddToDict(float);
             break;
         case R8:
-            SetDict1(double);
+            AddToDict(double);
             break;
         case TX:
-        {
-            PythonObject<string>* col = dynamic_cast<PythonObject<string>*>(column);
-            auto shrd = col->GetData();
-            bp::list list;
-            for (size_t i = 0; i < shrd->size(); i++)
-            {
-                bp::object obj;
-                const std::string& value = shrd->at(i);
-                if (!value.empty())
-                {
-                    obj = bp::object(value);
-                }
-                list.append(obj);
-            }
-            dict[_names[i]] = list;
+            AddToDict(std::string);
             delete column;
-        }
-        break;
+            break;
         case TS:
         case DT:
         case DZ:

src/NativeBridge/ManagedInterop.h

@@ -116,7 +116,7 @@ class CLASS_ALIGN EnvironmentBlock
     // there are no key names.
     std::vector<CxInt64> _columnToKeyMap;
 
-    std::vector<PythonObject<std::string>*> _vKeyValues;
+    std::vector<PythonObjectSingle<std::string>*> _vKeyValues;
 
     static MANAGED_CALLBACK(void) SetR4(EnvironmentBlock *env, int col, long index, float value)
     {