Skip to content
This repository was archived by the owner on Nov 16, 2023. It is now read-only.

Add variable length vector support #267

Merged
pieths merged 32 commits into from
Sep 30, 2019
Merged

Add variable length vector support #267

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
ae110ed
Add a template typed base PythonObject which provides the interface f…
Sep 12, 2019
da6ffd6
Add AddToDict method to the PythonObject interface.
Sep 12, 2019
c8cbf7b
Initial implementation of a PythonObject derived class which supports…
Sep 13, 2019
e4445ae
Select the type of PythonObject to create based on the column value c…
Sep 13, 2019
fda8a6c
Update bridge setter methods to take in both row and column rather th…
Sep 13, 2019
1b9ba54
Enable variable length columns for non-csr output data.
Sep 13, 2019
ef64291
Add column full of NaNs if no output value was ever specified.
Sep 13, 2019
9f20a35
Fix compile issues with the last checkin.
Sep 13, 2019
3e61f30
Merge branch 'master' into variable-length-vector
ganik Sep 17, 2019
5defe1f
Add proper PythonObjectVariable deleter.
Sep 17, 2019
41c98e4
Initial skeleton code for testing variable length column value support.
Sep 17, 2019
d303d1e
Merge remote-tracking branch 'origin/variable-length-vector' into var…
Sep 17, 2019
63bd45e
Add support for specifying vector lengths to VariableColumnTransform.
Sep 19, 2019
426c060
Fix exception message in VariableColumnTransform.
Sep 19, 2019
d52b60f
Add initial tests for the variable length output conversion.
Sep 19, 2019
13fa748
Force re-run of ci tests since linux build crashed on previous run.
Sep 19, 2019
bc224b9
Zero pad column names when converting variable length vector columns.
Sep 20, 2019
042a4ac
Merge remote-tracking branch 'upstream/master' into variable-length-v…
Sep 20, 2019
6e00598
Use the new entrypoints file in DotNetBridge for the VariableColumnTr…
Sep 20, 2019
3a1caf8
Merge branch 'master' into variable-length-vector
Sep 23, 2019
eb06ae3
Add more detailed comments regarding why integers are converted to fl…
Sep 25, 2019
3f0cde9
Merge branch 'master' into variable-length-vector
Sep 25, 2019
39bf5c0
Rename the PythonObject* classes to PyColumn* in NativeBridge.
Sep 25, 2019
c9b2884
Clean up indentation and comments.
Sep 25, 2019
4b129d4
Refactor PyColumnVariable to support different behaviors for other ty…
Sep 26, 2019
15a474d
Add string column type support.
Sep 26, 2019
823ba3e
Merge branch 'master' into variable-length-vector
Sep 27, 2019
95e28f3
Fix int32 to float32 conversion losses precision.
Sep 27, 2019
e97079e
Clean up pointer usage in PyColumnVariable.
Sep 27, 2019
20dda09
Remove numCols from creation of PyColumn based classes.
Sep 27, 2019
0b3c993
Remove unnecessary assembly attribute.
Sep 27, 2019
1facdf5
Set initial capacity of valueCounts list in NativeDataInterop.
Sep 27, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 12 additions & 12 deletions src/DotNetBridge/Bridge.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,51 +128,51 @@ public unsafe static partial class Bridge

// For setting bool values to NativeBridge.
[UnmanagedFunctionPointer(CallingConvention.StdCall)]
private unsafe delegate void BLSetter(EnvironmentBlock* penv, int col, long index, byte value);
private unsafe delegate void BLSetter(EnvironmentBlock* penv, int col, long m, long n, byte value);

// For setting float values to NativeBridge.
[UnmanagedFunctionPointer(CallingConvention.StdCall)]
private unsafe delegate void R4Setter(EnvironmentBlock* penv, int col, long index, float value);
private unsafe delegate void R4Setter(EnvironmentBlock* penv, int col, long m, long n, float value);

// For setting double values to NativeBridge.
[UnmanagedFunctionPointer(CallingConvention.StdCall)]
private unsafe delegate void R8Setter(EnvironmentBlock* penv, int col, long index, double value);
private unsafe delegate void R8Setter(EnvironmentBlock* penv, int col, long m, long n, double value);

// For setting I1 values to NativeBridge.
[UnmanagedFunctionPointer(CallingConvention.StdCall)]
private unsafe delegate void I1Setter(EnvironmentBlock* penv, int col, long index, sbyte value);
private unsafe delegate void I1Setter(EnvironmentBlock* penv, int col, long m, long n, sbyte value);

// For setting I2 values to NativeBridge.
[UnmanagedFunctionPointer(CallingConvention.StdCall)]
private unsafe delegate void I2Setter(EnvironmentBlock* penv, int col, long index, short value);
private unsafe delegate void I2Setter(EnvironmentBlock* penv, int col, long m, long n, short value);

// For setting I4 values to NativeBridge.
[UnmanagedFunctionPointer(CallingConvention.StdCall)]
private unsafe delegate void I4Setter(EnvironmentBlock* penv, int col, long index, int value);
private unsafe delegate void I4Setter(EnvironmentBlock* penv, int col, long m, long n, int value);

// For setting I8 values to NativeBridge.
[UnmanagedFunctionPointer(CallingConvention.StdCall)]
private unsafe delegate void I8Setter(EnvironmentBlock* penv, int col, long index, long value);
private unsafe delegate void I8Setter(EnvironmentBlock* penv, int col, long m, long n, long value);

// For setting U1 values to NativeBridge.
[UnmanagedFunctionPointer(CallingConvention.StdCall)]
private unsafe delegate void U1Setter(EnvironmentBlock* penv, int col, long index, byte value);
private unsafe delegate void U1Setter(EnvironmentBlock* penv, int col, long m, long n, byte value);

// For setting U2 values to NativeBridge.
[UnmanagedFunctionPointer(CallingConvention.StdCall)]
private unsafe delegate void U2Setter(EnvironmentBlock* penv, int col, long index, ushort value);
private unsafe delegate void U2Setter(EnvironmentBlock* penv, int col, long m, long n, ushort value);

// For setting U4 values to NativeBridge.
[UnmanagedFunctionPointer(CallingConvention.StdCall)]
private unsafe delegate void U4Setter(EnvironmentBlock* penv, int col, long index, uint value);
private unsafe delegate void U4Setter(EnvironmentBlock* penv, int col, long m, long n, uint value);

// For setting U8 values to NativeBridge.
[UnmanagedFunctionPointer(CallingConvention.StdCall)]
private unsafe delegate void U8Setter(EnvironmentBlock* penv, int col, long index, ulong value);
private unsafe delegate void U8Setter(EnvironmentBlock* penv, int col, long m, long n, ulong value);

// For setting string values, to a generic pointer and index.
[UnmanagedFunctionPointer(CallingConvention.StdCall)]
private unsafe delegate void TXSetter(EnvironmentBlock* penv, int col, long index, sbyte* pch, int cch);
private unsafe delegate void TXSetter(EnvironmentBlock* penv, int col, long m, long n, sbyte* pch, int cch);

// For setting string key values, to a generic pointer and index.
[UnmanagedFunctionPointer(CallingConvention.StdCall)]
Expand Down
15 changes: 15 additions & 0 deletions src/DotNetBridge/Entrypoints.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@

[assembly: LoadableClass(typeof(void), typeof(DotNetBridgeEntrypoints), null, typeof(SignatureEntryPointModule), "DotNetBridgeEntrypoints")]

[assembly: LoadableClass(VariableColumnTransform.Summary, typeof(VariableColumnTransform), null, typeof(SignatureLoadDataTransform),
"", VariableColumnTransform.LoaderSignature)]

namespace Microsoft.ML.DotNetBridge
{
internal static class DotNetBridgeEntrypoints
Expand Down Expand Up @@ -72,5 +75,17 @@ public static ModelSchemaOutput GetSchema(IHostEnvironment env, TransformModelIn

return new ModelSchemaOutput { Schema = new EmptyDataView(host, input.Model.OutputSchema) };
}

[TlcModule.EntryPoint(Name = "Transforms.VariableColumnTransform", Desc = VariableColumnTransform.Summary,
UserName = "Variable Column Creator", ShortName = "Variable Column Creator")]
public static CommonOutputs.TransformOutput CreateVariableColumn(IHostEnvironment env, VariableColumnTransform.Options inputOptions)
{
Contracts.CheckValue(env, nameof(env));
var host = env.Register("VariableColumnCreator");
EntryPointUtils.CheckInputArgs(host, inputOptions);

var xf = VariableColumnTransform.Create(env, inputOptions, inputOptions.Data);
return new CommonOutputs.TransformOutput { Model = new TransformModelImpl(env, xf, inputOptions.Data), OutputData = xf };
}
}
}
Loading