Skip to content

Adding basic support for handling vectors in the database loader. #4138

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
eerhardt merged 15 commits into from
Aug 30, 2019
Merged

Adding basic support for handling vectors in the database loader. #4138

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
b72a7b5
Adding basic support for handling vectors in the database loader.
tannergooding Aug 21, 2019
794c207
Updating the DatabaseLoaderTests to run against an actual database.
tannergooding Aug 27, 2019
d21174a
Fixing a variable name to avoid a conflict
tannergooding Aug 28, 2019
cef65eb
Fixing the DatabaseLoaderTests.IrisSdcaMaximumEntropy test to look in…
tannergooding Aug 28, 2019
0a50703
Adding back a type check, returning NaN as the default for float/doub…
tannergooding Aug 28, 2019
4cba514
Fixing up the DatabaseLoader tests to use the Iris TestDatabase
tannergooding Aug 29, 2019
cd59439
Remove a call to pipeline.Preview
tannergooding Aug 29, 2019
1b9ee0a
Fix the name of loaderColumns1 to loaderColumns
tannergooding Aug 29, 2019
712ad44
Fixing up doc comments and removing allocations from the database loa…
tannergooding Aug 29, 2019
d809d8f
Responding to PR feedback and removing dead code.
tannergooding Aug 29, 2019
64d9553
Mark the DatabaseLoaderTests as Windows specific for now.
tannergooding Aug 29, 2019
54b7a99
Fixing the context writer to not nullref for a null segment
tannergooding Aug 29, 2019
9da2bd6
Ensure the Bindings segments is null if the count is zero.
tannergooding Aug 29, 2019
b5c1e7b
Fix tests to build on netfx, and to be skipped on non-Windows.
eerhardt Aug 29, 2019
f611b49
Adding a connection timeout to give localdb a chance to initialize
tannergooding Aug 30, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions build/Dependencies.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,11 @@
<PropertyGroup>
<BenchmarkDotNetVersion>0.11.3</BenchmarkDotNetVersion>
<MicrosoftCodeAnalysisTestingVersion>1.0.0-beta1-63812-02</MicrosoftCodeAnalysisTestingVersion>
<MicrosoftMLTestDatabasesPackageVersion>0.0.5-test</MicrosoftMLTestDatabasesPackageVersion>
<MicrosoftMLTestModelsPackageVersion>0.0.5-test</MicrosoftMLTestModelsPackageVersion>
<MicrosoftMLTensorFlowTestModelsVersion>0.0.11-test</MicrosoftMLTensorFlowTestModelsVersion>
<MicrosoftMLOnnxTestModelsVersion>0.0.5-test</MicrosoftMLOnnxTestModelsVersion>
<SystemDataSqlClientVersion>4.6.1</SystemDataSqlClientVersion>
</PropertyGroup>

</Project>
290 changes: 223 additions & 67 deletions src/Microsoft.ML.Experimental/DataLoadSave/Database/DatabaseLoader.cs
View file
Open in desktop

Large diffs are not rendered by default.

352 changes: 351 additions & 1 deletion src/Microsoft.ML.Experimental/DataLoadSave/Database/DatabaseLoaderCursor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,58 @@ private Delegate CreateGetterDelegate<TValue>(int col)
{
getterDelegate = CreateUInt64GetterDelegate(colInfo);
}
else if (typeof(TValue) == typeof(VBuffer<bool>))
{
getterDelegate = CreateVBufferBooleanGetterDelegate(colInfo);
}
else if (typeof(TValue) == typeof(VBuffer<byte>))
{
getterDelegate = CreateVBufferByteGetterDelegate(colInfo);
}
else if (typeof(TValue) == typeof(VBuffer<DateTime>))
{
getterDelegate = CreateVBufferDateTimeGetterDelegate(colInfo);
}
else if (typeof(TValue) == typeof(VBuffer<double>))
{
getterDelegate = CreateVBufferDoubleGetterDelegate(colInfo);
}
else if (typeof(TValue) == typeof(VBuffer<short>))
{
getterDelegate = CreateVBufferInt16GetterDelegate(colInfo);
}
else if (typeof(TValue) == typeof(VBuffer<int>))
{
getterDelegate = CreateVBufferInt32GetterDelegate(colInfo);
}
else if (typeof(TValue) == typeof(VBuffer<long>))
{
getterDelegate = CreateVBufferInt64GetterDelegate(colInfo);
}
else if (typeof(TValue) == typeof(VBuffer<sbyte>))
{
getterDelegate = CreateVBufferSByteGetterDelegate(colInfo);
}
else if (typeof(TValue) == typeof(VBuffer<float>))
{
getterDelegate = CreateVBufferSingleGetterDelegate(colInfo);
}
else if (typeof(TValue) == typeof(VBuffer<ReadOnlyMemory<char>>))
{
getterDelegate = CreateVBufferStringGetterDelegate(colInfo);
}
else if (typeof(TValue) == typeof(VBuffer<ushort>))
{
getterDelegate = CreateVBufferUInt16GetterDelegate(colInfo);
}
else if (typeof(TValue) == typeof(VBuffer<uint>))
{
getterDelegate = CreateVBufferUInt32GetterDelegate(colInfo);
}
else if (typeof(TValue) == typeof(VBuffer<ulong>))
{
getterDelegate = CreateVBufferUInt64GetterDelegate(colInfo);
}
else
{
throw new NotSupportedException();
Expand Down Expand Up @@ -311,9 +363,307 @@ private ValueGetter<ulong> CreateUInt64GetterDelegate(ColInfo colInfo)
return (ref ulong value) => value = DataReader.IsDBNull(columnIndex) ? default : (ulong)DataReader.GetInt64(columnIndex);
}

private ValueGetter<VBuffer<bool>> CreateVBufferBooleanGetterDelegate(ColInfo colInfo)
{
return (ref VBuffer<bool> value) =>
{
int length = colInfo.SizeBase;
bool[] values = new bool[length];
Comment thread
tannergooding marked this conversation as resolved.
Outdated

int i = 0;
var segs = colInfo.Segments;

foreach (var seg in segs)
{
for (int columnIndex = seg.Min; columnIndex < seg.Lim; columnIndex++, i++)
{
values[i] = DataReader.IsDBNull(columnIndex) ? default : DataReader.GetBoolean(columnIndex);
}
}

value = new VBuffer<bool>(length, values);
};
}

private ValueGetter<VBuffer<byte>> CreateVBufferByteGetterDelegate(ColInfo colInfo)
{
return (ref VBuffer<byte> value) =>
{
int length = colInfo.SizeBase;
byte[] values = new byte[length];

int i = 0;
var segs = colInfo.Segments;

foreach (var seg in segs)
{
for (int columnIndex = seg.Min; columnIndex < seg.Lim; columnIndex++, i++)
{
values[i] = DataReader.IsDBNull(columnIndex) ? default : DataReader.GetByte(columnIndex);
}
}

value = new VBuffer<byte>(length, values);
};
}

private ValueGetter<VBuffer<DateTime>> CreateVBufferDateTimeGetterDelegate(ColInfo colInfo)
{
return (ref VBuffer<DateTime> value) =>
{
int length = colInfo.SizeBase;
DateTime[] values = new DateTime[length];

int i = 0;
var segs = colInfo.Segments;

foreach (var seg in segs)
{
for (int columnIndex = seg.Min; columnIndex < seg.Lim; columnIndex++, i++)
{
values[i] = DataReader.IsDBNull(columnIndex) ? default : DataReader.GetDateTime(columnIndex);
}
}

value = new VBuffer<DateTime>(length, values);
};
}

private ValueGetter<VBuffer<double>> CreateVBufferDoubleGetterDelegate(ColInfo colInfo)
{
return (ref VBuffer<double> value) =>
{
int length = colInfo.SizeBase;
double[] values = new double[length];

int i = 0;
var segs = colInfo.Segments;

foreach (var seg in segs)
{
for (int columnIndex = seg.Min; columnIndex < seg.Lim; columnIndex++, i++)
{
values[i] = DataReader.IsDBNull(columnIndex) ? double.NaN : DataReader.GetDouble(columnIndex);
}
}

value = new VBuffer<double>(length, values);
};
}

private ValueGetter<VBuffer<short>> CreateVBufferInt16GetterDelegate(ColInfo colInfo)
{
return (ref VBuffer<short> value) =>
{
int length = colInfo.SizeBase;
short[] values = new short[length];

int i = 0;
var segs = colInfo.Segments;

foreach (var seg in segs)
{
for (int columnIndex = seg.Min; columnIndex < seg.Lim; columnIndex++, i++)
{
values[i] = DataReader.IsDBNull(columnIndex) ? default : DataReader.GetInt16(columnIndex);
}
}

value = new VBuffer<short>(length, values);
};
}

private ValueGetter<VBuffer<int>> CreateVBufferInt32GetterDelegate(ColInfo colInfo)
{
return (ref VBuffer<int> value) =>
{
int length = colInfo.SizeBase;
int[] values = new int[length];

int i = 0;
var segs = colInfo.Segments;

foreach (var seg in segs)
{
for (int columnIndex = seg.Min; columnIndex < seg.Lim; columnIndex++, i++)
{
values[i] = DataReader.IsDBNull(columnIndex) ? default : DataReader.GetInt32(columnIndex);
}
}

value = new VBuffer<int>(length, values);
};
}

private ValueGetter<VBuffer<long>> CreateVBufferInt64GetterDelegate(ColInfo colInfo)
{
return (ref VBuffer<long> value) =>
{
int length = colInfo.SizeBase;
long[] values = new long[length];

int i = 0;
var segs = colInfo.Segments;

foreach (var seg in segs)
{
for (int columnIndex = seg.Min; columnIndex < seg.Lim; columnIndex++, i++)
{
values[i] = DataReader.IsDBNull(columnIndex) ? default : DataReader.GetInt64(columnIndex);
}
}

value = new VBuffer<long>(length, values);
};
}

private ValueGetter<VBuffer<sbyte>> CreateVBufferSByteGetterDelegate(ColInfo colInfo)
{
return (ref VBuffer<sbyte> value) =>
{
int length = colInfo.SizeBase;
sbyte[] values = new sbyte[length];

int i = 0;
var segs = colInfo.Segments;

foreach (var seg in segs)
{
for (int columnIndex = seg.Min; columnIndex < seg.Lim; columnIndex++, i++)
{
values[i] = DataReader.IsDBNull(columnIndex) ? default : (sbyte)DataReader.GetByte(columnIndex);
}
}

value = new VBuffer<sbyte>(length, values);
};
}

private ValueGetter<VBuffer<float>> CreateVBufferSingleGetterDelegate(ColInfo colInfo)
{
return (ref VBuffer<float> value) =>
{
int length = colInfo.SizeBase;
float[] values = new float[length];

int i = 0;
var segs = colInfo.Segments;

foreach (var seg in segs)
{
for (int columnIndex = seg.Min; columnIndex < seg.Lim; columnIndex++, i++)
{
values[i] = DataReader.IsDBNull(columnIndex) ? float.NaN : DataReader.GetFloat(columnIndex);
}
}

value = new VBuffer<float>(length, values);
};
}

private ValueGetter<VBuffer<ReadOnlyMemory<char>>> CreateVBufferStringGetterDelegate(ColInfo colInfo)
{
return (ref VBuffer<ReadOnlyMemory<char>> value) =>
{
int length = colInfo.SizeBase;
ReadOnlyMemory<char>[] values = new ReadOnlyMemory<char>[length];

int i = 0;
var segs = colInfo.Segments;

foreach (var seg in segs)
{
for (int columnIndex = seg.Min; columnIndex < seg.Lim; columnIndex++, i++)
{
values[i] = DataReader.IsDBNull(columnIndex) ? default : DataReader.GetString(columnIndex).AsMemory();
}
}

value = new VBuffer<ReadOnlyMemory<char>>(length, values);
};
}

private ValueGetter<VBuffer<ushort>> CreateVBufferUInt16GetterDelegate(ColInfo colInfo)
{
return (ref VBuffer<ushort> value) =>
{
int length = colInfo.SizeBase;
ushort[] values = new ushort[length];

int i = 0;
var segs = colInfo.Segments;

foreach (var seg in segs)
{
for (int columnIndex = seg.Min; columnIndex < seg.Lim; columnIndex++, i++)
{
values[i] = DataReader.IsDBNull(columnIndex) ? default : (ushort)DataReader.GetInt16(columnIndex);
}
}

value = new VBuffer<ushort>(length, values);
};
}

private ValueGetter<VBuffer<uint>> CreateVBufferUInt32GetterDelegate(ColInfo colInfo)
{
return (ref VBuffer<uint> value) =>
{
int length = colInfo.SizeBase;
uint[] values = new uint[length];

int i = 0;
var segs = colInfo.Segments;

foreach (var seg in segs)
{
for (int columnIndex = seg.Min; columnIndex < seg.Lim; columnIndex++, i++)
{
values[i] = DataReader.IsDBNull(columnIndex) ? default : (uint)DataReader.GetInt32(columnIndex);
}
}

value = new VBuffer<uint>(length, values);
};
}

private ValueGetter<VBuffer<ulong>> CreateVBufferUInt64GetterDelegate(ColInfo colInfo)
{
return (ref VBuffer<ulong> value) =>
{
int length = colInfo.SizeBase;
ulong[] values = new ulong[length];

int i = 0;
var segs = colInfo.Segments;

foreach (var seg in segs)
{
for (int columnIndex = seg.Min; columnIndex < seg.Lim; columnIndex++, i++)
{
values[i] = DataReader.IsDBNull(columnIndex) ? default : (ulong)DataReader.GetInt64(columnIndex);
}
}

value = new VBuffer<ulong>(length, values);
};
}

private int GetColumnIndex(ColInfo colInfo)
{
return colInfo.SourceIndex ?? DataReader.GetOrdinal(colInfo.Name);
var segs = colInfo.Segments;

if (segs is null)
{
return DataReader.GetOrdinal(colInfo.Name);
}

Contracts.Check(segs.Length == 1);

var seg = segs[0];
Contracts.Check(seg.Min == seg.Lim);

return seg.Min;
}
}
}
Expand Down
Loading