Skip to content

Adding basic support for handling vectors in the database loader. #4138

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
eerhardt merged 15 commits into from
Aug 30, 2019
Merged

Adding basic support for handling vectors in the database loader. #4138

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
b72a7b5
Adding basic support for handling vectors in the database loader.
tannergooding Aug 21, 2019
794c207
Updating the DatabaseLoaderTests to run against an actual database.
tannergooding Aug 27, 2019
d21174a
Fixing a variable name to avoid a conflict
tannergooding Aug 28, 2019
cef65eb
Fixing the DatabaseLoaderTests.IrisSdcaMaximumEntropy test to look in…
tannergooding Aug 28, 2019
0a50703
Adding back a type check, returning NaN as the default for float/doub…
tannergooding Aug 28, 2019
4cba514
Fixing up the DatabaseLoader tests to use the Iris TestDatabase
tannergooding Aug 29, 2019
cd59439
Remove a call to pipeline.Preview
tannergooding Aug 29, 2019
1b9ee0a
Fix the name of loaderColumns1 to loaderColumns
tannergooding Aug 29, 2019
712ad44
Fixing up doc comments and removing allocations from the database loa…
tannergooding Aug 29, 2019
d809d8f
Responding to PR feedback and removing dead code.
tannergooding Aug 29, 2019
64d9553
Mark the DatabaseLoaderTests as Windows specific for now.
tannergooding Aug 29, 2019
54b7a99
Fixing the context writer to not nullref for a null segment
tannergooding Aug 29, 2019
9da2bd6
Ensure the Bindings segments is null if the count is zero.
tannergooding Aug 29, 2019
b5c1e7b
Fix tests to build on netfx, and to be skipped on non-Windows.
eerhardt Aug 29, 2019
f611b49
Adding a connection timeout to give localdb a chance to initialize
tannergooding Aug 30, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions build/Dependencies.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,11 @@
<PropertyGroup>
<BenchmarkDotNetVersion>0.11.3</BenchmarkDotNetVersion>
<MicrosoftCodeAnalysisTestingVersion>1.0.0-beta1-63812-02</MicrosoftCodeAnalysisTestingVersion>
<MicrosoftMLTestDatabasesPackageVersion>0.0.5-test</MicrosoftMLTestDatabasesPackageVersion>
<MicrosoftMLTestModelsPackageVersion>0.0.5-test</MicrosoftMLTestModelsPackageVersion>
<MicrosoftMLTensorFlowTestModelsVersion>0.0.11-test</MicrosoftMLTensorFlowTestModelsVersion>
<MicrosoftMLOnnxTestModelsVersion>0.0.5-test</MicrosoftMLOnnxTestModelsVersion>
<SystemDataSqlClientVersion>4.6.1</SystemDataSqlClientVersion>
</PropertyGroup>

</Project>
267 changes: 202 additions & 65 deletions src/Microsoft.ML.Experimental/DataLoadSave/Database/DatabaseLoader.cs
View file
Open in desktop

Large diffs are not rendered by default.

352 changes: 351 additions & 1 deletion src/Microsoft.ML.Experimental/DataLoadSave/Database/DatabaseLoaderCursor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,58 @@ private Delegate CreateGetterDelegate<TValue>(int col)
{
getterDelegate = CreateUInt64GetterDelegate(colInfo);
}
else if (typeof(TValue) == typeof(VBuffer<bool>))
{
getterDelegate = CreateVBufferBooleanGetterDelegate(colInfo);
}
else if (typeof(TValue) == typeof(VBuffer<byte>))
{
getterDelegate = CreateVBufferByteGetterDelegate(colInfo);
}
else if (typeof(TValue) == typeof(VBuffer<DateTime>))
{
getterDelegate = CreateVBufferDateTimeGetterDelegate(colInfo);
}
else if (typeof(TValue) == typeof(VBuffer<double>))
{
getterDelegate = CreateVBufferDoubleGetterDelegate(colInfo);
}
else if (typeof(TValue) == typeof(VBuffer<short>))
{
getterDelegate = CreateVBufferInt16GetterDelegate(colInfo);
}
else if (typeof(TValue) == typeof(VBuffer<int>))
{
getterDelegate = CreateVBufferInt32GetterDelegate(colInfo);
}
else if (typeof(TValue) == typeof(VBuffer<long>))
{
getterDelegate = CreateVBufferInt64GetterDelegate(colInfo);
}
else if (typeof(TValue) == typeof(VBuffer<sbyte>))
{
getterDelegate = CreateVBufferSByteGetterDelegate(colInfo);
}
else if (typeof(TValue) == typeof(VBuffer<float>))
{
getterDelegate = CreateVBufferSingleGetterDelegate(colInfo);
}
else if (typeof(TValue) == typeof(VBuffer<ReadOnlyMemory<char>>))
{
getterDelegate = CreateVBufferStringGetterDelegate(colInfo);
}
else if (typeof(TValue) == typeof(VBuffer<ushort>))
{
getterDelegate = CreateVBufferUInt16GetterDelegate(colInfo);
}
else if (typeof(TValue) == typeof(VBuffer<uint>))
{
getterDelegate = CreateVBufferUInt32GetterDelegate(colInfo);
}
else if (typeof(TValue) == typeof(VBuffer<ulong>))
{
getterDelegate = CreateVBufferUInt64GetterDelegate(colInfo);
}
else
{
throw new NotSupportedException();
Expand Down Expand Up @@ -311,9 +363,307 @@ private ValueGetter<ulong> CreateUInt64GetterDelegate(ColInfo colInfo)
return (ref ulong value) => value = DataReader.IsDBNull(columnIndex) ? default : (ulong)DataReader.GetInt64(columnIndex);
}

private ValueGetter<VBuffer<bool>> CreateVBufferBooleanGetterDelegate(ColInfo colInfo)
{
return (ref VBuffer<bool> value) =>
{
int length = colInfo.SizeBase;
var editor = VBufferEditor.Create(ref value, length);

int i = 0;
var segs = colInfo.Segments;

foreach (var seg in segs)
{
for (int columnIndex = seg.Min; columnIndex < seg.Lim; columnIndex++, i++)
{
editor.Values[i] = DataReader.IsDBNull(columnIndex) ? default : DataReader.GetBoolean(columnIndex);
}
}

value = editor.Commit();
};
}

private ValueGetter<VBuffer<byte>> CreateVBufferByteGetterDelegate(ColInfo colInfo)
{
return (ref VBuffer<byte> value) =>
{
int length = colInfo.SizeBase;
var editor = VBufferEditor.Create(ref value, length);

int i = 0;
var segs = colInfo.Segments;

foreach (var seg in segs)
{
for (int columnIndex = seg.Min; columnIndex < seg.Lim; columnIndex++, i++)
{
editor.Values[i] = DataReader.IsDBNull(columnIndex) ? default : DataReader.GetByte(columnIndex);
}
}

value = editor.Commit();
};
}

private ValueGetter<VBuffer<DateTime>> CreateVBufferDateTimeGetterDelegate(ColInfo colInfo)
{
return (ref VBuffer<DateTime> value) =>
{
int length = colInfo.SizeBase;
var editor = VBufferEditor.Create(ref value, length);

int i = 0;
var segs = colInfo.Segments;

foreach (var seg in segs)
{
for (int columnIndex = seg.Min; columnIndex < seg.Lim; columnIndex++, i++)
{
editor.Values[i] = DataReader.IsDBNull(columnIndex) ? default : DataReader.GetDateTime(columnIndex);
}
}

value = editor.Commit();
};
}

private ValueGetter<VBuffer<double>> CreateVBufferDoubleGetterDelegate(ColInfo colInfo)
{
return (ref VBuffer<double> value) =>
{
int length = colInfo.SizeBase;
var editor = VBufferEditor.Create(ref value, length);

int i = 0;
var segs = colInfo.Segments;

foreach (var seg in segs)
{
for (int columnIndex = seg.Min; columnIndex < seg.Lim; columnIndex++, i++)
{
editor.Values[i] = DataReader.IsDBNull(columnIndex) ? double.NaN : DataReader.GetDouble(columnIndex);
}
}

value = editor.Commit();
};
}

private ValueGetter<VBuffer<short>> CreateVBufferInt16GetterDelegate(ColInfo colInfo)
{
return (ref VBuffer<short> value) =>
{
int length = colInfo.SizeBase;
var editor = VBufferEditor.Create(ref value, length);

int i = 0;
var segs = colInfo.Segments;

foreach (var seg in segs)
{
for (int columnIndex = seg.Min; columnIndex < seg.Lim; columnIndex++, i++)
{
editor.Values[i] = DataReader.IsDBNull(columnIndex) ? default : DataReader.GetInt16(columnIndex);
}
}

value = editor.Commit();
};
}

private ValueGetter<VBuffer<int>> CreateVBufferInt32GetterDelegate(ColInfo colInfo)
{
return (ref VBuffer<int> value) =>
{
int length = colInfo.SizeBase;
var editor = VBufferEditor.Create(ref value, length);

int i = 0;
var segs = colInfo.Segments;

foreach (var seg in segs)
{
for (int columnIndex = seg.Min; columnIndex < seg.Lim; columnIndex++, i++)
{
editor.Values[i] = DataReader.IsDBNull(columnIndex) ? default : DataReader.GetInt32(columnIndex);
}
}

value = editor.Commit();
};
}

private ValueGetter<VBuffer<long>> CreateVBufferInt64GetterDelegate(ColInfo colInfo)
{
return (ref VBuffer<long> value) =>
{
int length = colInfo.SizeBase;
var editor = VBufferEditor.Create(ref value, length);

int i = 0;
var segs = colInfo.Segments;

foreach (var seg in segs)
{
for (int columnIndex = seg.Min; columnIndex < seg.Lim; columnIndex++, i++)
{
editor.Values[i] = DataReader.IsDBNull(columnIndex) ? default : DataReader.GetInt64(columnIndex);
}
}

value = editor.Commit();
};
}

private ValueGetter<VBuffer<sbyte>> CreateVBufferSByteGetterDelegate(ColInfo colInfo)
{
return (ref VBuffer<sbyte> value) =>
{
int length = colInfo.SizeBase;
var editor = VBufferEditor.Create(ref value, length);

int i = 0;
var segs = colInfo.Segments;

foreach (var seg in segs)
{
for (int columnIndex = seg.Min; columnIndex < seg.Lim; columnIndex++, i++)
{
editor.Values[i] = DataReader.IsDBNull(columnIndex) ? default : (sbyte)DataReader.GetByte(columnIndex);
}
}

value = editor.Commit();
};
}

private ValueGetter<VBuffer<float>> CreateVBufferSingleGetterDelegate(ColInfo colInfo)
{
return (ref VBuffer<float> value) =>
{
int length = colInfo.SizeBase;
var editor = VBufferEditor.Create(ref value, length);

int i = 0;
var segs = colInfo.Segments;

foreach (var seg in segs)
{
for (int columnIndex = seg.Min; columnIndex < seg.Lim; columnIndex++, i++)
{
editor.Values[i] = DataReader.IsDBNull(columnIndex) ? float.NaN : DataReader.GetFloat(columnIndex);
}
}

value = editor.Commit();
};
}

private ValueGetter<VBuffer<ReadOnlyMemory<char>>> CreateVBufferStringGetterDelegate(ColInfo colInfo)
{
return (ref VBuffer<ReadOnlyMemory<char>> value) =>
{
int length = colInfo.SizeBase;
var editor = VBufferEditor.Create(ref value, length);

int i = 0;
var segs = colInfo.Segments;

foreach (var seg in segs)
{
for (int columnIndex = seg.Min; columnIndex < seg.Lim; columnIndex++, i++)
{
editor.Values[i] = DataReader.IsDBNull(columnIndex) ? default : DataReader.GetString(columnIndex).AsMemory();
}
}

value = editor.Commit();
};
}

private ValueGetter<VBuffer<ushort>> CreateVBufferUInt16GetterDelegate(ColInfo colInfo)
{
return (ref VBuffer<ushort> value) =>
{
int length = colInfo.SizeBase;
var editor = VBufferEditor.Create(ref value, length);

int i = 0;
var segs = colInfo.Segments;

foreach (var seg in segs)
{
for (int columnIndex = seg.Min; columnIndex < seg.Lim; columnIndex++, i++)
{
editor.Values[i] = DataReader.IsDBNull(columnIndex) ? default : (ushort)DataReader.GetInt16(columnIndex);
}
}

value = editor.Commit();
};
}

private ValueGetter<VBuffer<uint>> CreateVBufferUInt32GetterDelegate(ColInfo colInfo)
{
return (ref VBuffer<uint> value) =>
{
int length = colInfo.SizeBase;
var editor = VBufferEditor.Create(ref value, length);

int i = 0;
var segs = colInfo.Segments;

foreach (var seg in segs)
{
for (int columnIndex = seg.Min; columnIndex < seg.Lim; columnIndex++, i++)
{
editor.Values[i] = DataReader.IsDBNull(columnIndex) ? default : (uint)DataReader.GetInt32(columnIndex);
}
}

value = editor.Commit();
};
}

private ValueGetter<VBuffer<ulong>> CreateVBufferUInt64GetterDelegate(ColInfo colInfo)
{
return (ref VBuffer<ulong> value) =>
{
int length = colInfo.SizeBase;
var editor = VBufferEditor.Create(ref value, length);

int i = 0;
var segs = colInfo.Segments;

foreach (var seg in segs)
{
for (int columnIndex = seg.Min; columnIndex < seg.Lim; columnIndex++, i++)
{
editor.Values[i] = DataReader.IsDBNull(columnIndex) ? default : (ulong)DataReader.GetInt64(columnIndex);
}
}

value = editor.Commit();
};
}

private int GetColumnIndex(ColInfo colInfo)
{
return colInfo.SourceIndex ?? DataReader.GetOrdinal(colInfo.Name);
var segs = colInfo.Segments;

if (segs is null)
{
return DataReader.GetOrdinal(colInfo.Name);
}

Contracts.Check(segs.Length == 1);

var seg = segs[0];
Contracts.Check(seg.Min == seg.Lim);

return seg.Min;
}
}
}
Expand Down
Loading