Skip to content

Don't fail in case of const field in Collection source and extended support for type conversion #555

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Ivanidzo4ka merged 10 commits into from
Jul 23, 2018
Merged

Don't fail in case of const field in Collection source and extended support for type conversion #555

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
e8ea5c4
Don't fail in case of const field in Collection source.
Jul 19, 2018
581cb42
fix tests, address some comments
Jul 19, 2018
62091c9
support nullable arrays.
Jul 19, 2018
ef950eb
bool case
Jul 19, 2018
49323f2
clean test code
Jul 19, 2018
d556d13
address comments
Jul 19, 2018
a7bb6e1
more comments!
Jul 19, 2018
d5143e6
give functions are better name?
Jul 19, 2018
5b7dc4a
add //Review comment regarding conversion
Jul 19, 2018
5f69a4c
address pete comments and format tests properly
Jul 19, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions src/Microsoft.ML.Api/ApiUtils.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,10 @@ private static OpCode GetAssignmentOpCode(Type t)
// REVIEW: This should be a Dictionary<Type, OpCode> based solution.
// DvTexts, strings, arrays, and VBuffers.
if (t == typeof(DvInt8) || t == typeof(DvInt4) || t == typeof(DvInt2) || t == typeof(DvInt1) ||

@Zruty0 Zruty0 Jul 19, 2018 •
edited
Loading

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fix the comment #Closed

t == typeof(DvBool) || t==typeof(bool?) || t == typeof(DvText) || t == typeof(string) || t.IsArray ||
(t.IsGenericType && t.GetGenericTypeDefinition() == typeof(VBuffer<>)) || t == typeof(DvDateTime) ||
t == typeof(DvDateTimeZone) || t == typeof(DvTimeSpan) || t == typeof(UInt128))
t == typeof(DvBool) || t == typeof(DvText) || t == typeof(string) || t.IsArray ||
(t.IsGenericType && t.GetGenericTypeDefinition() == typeof(VBuffer<>)) ||
(t.IsGenericType && t.GetGenericTypeDefinition() == typeof(Nullable<>)) ||
t == typeof(DvDateTime) || t == typeof(DvDateTimeZone) || t == typeof(DvTimeSpan) || t == typeof(UInt128))
{
return OpCodes.Stobj;
}
Expand Down
205 changes: 144 additions & 61 deletions src/Microsoft.ML.Api/DataViewConstructionUtils.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ private Delegate CreateGetter(int index)

var column = DataView._schema.SchemaDefn.Columns[index];
var outputType = column.IsComputed ? column.ReturnType : column.FieldInfo.FieldType;

var genericType = outputType;
Func<int, Delegate> del;

if (outputType.IsArray)
Expand All @@ -129,112 +129,192 @@ private Delegate CreateGetter(int index)
if (outputType.GetElementType() == typeof(string))
{
Ch.Assert(colType.ItemType.IsText);
return CreateStringArrayToVBufferGetter(index);
return CreateArrayGetterDelegate<String, DvText>(index, x => new DvText(x));
}
else if (outputType.GetElementType() == typeof(int))
{
Ch.Assert(colType.ItemType == NumberType.I4);
return CreateArrayGetterDelegate<int, DvInt4>(index, x => x);
}
else if (outputType.GetElementType() == typeof(int?))
{
Ch.Assert(colType.ItemType == NumberType.I4);
return CreateArrayGetterDelegate<int?, DvInt4>(index, x => x ?? DvInt4.NA);
}
else if (outputType.GetElementType() == typeof(long))
{
Ch.Assert(colType.ItemType == NumberType.I8);
return CreateArrayGetterDelegate<long, DvInt8>(index, x => x);
}
else if (outputType.GetElementType() == typeof(long?))
{
Ch.Assert(colType.ItemType == NumberType.I8);
return CreateArrayGetterDelegate<long?, DvInt8>(index, x => x ?? DvInt8.NA);
}
else if (outputType.GetElementType() == typeof(short))
{
Ch.Assert(colType.ItemType == NumberType.I2);
return CreateArrayGetterDelegate<short, DvInt2>(index, x => x);
}
else if (outputType.GetElementType() == typeof(short?))
{
Ch.Assert(colType.ItemType == NumberType.I2);
return CreateArrayGetterDelegate<short?, DvInt2>(index, x => x ?? DvInt2.NA);
}
else if (outputType.GetElementType() == typeof(sbyte))
{
Ch.Assert(colType.ItemType == NumberType.I1);
return CreateArrayGetterDelegate<sbyte, DvInt1>(index, x => x);
}
else if (outputType.GetElementType() == typeof(sbyte?))
{
Ch.Assert(colType.ItemType == NumberType.I1);
return CreateArrayGetterDelegate<sbyte?, DvInt1>(index, x => x ?? DvInt1.NA);
}
else if (outputType.GetElementType() == typeof(bool))
{
Ch.Assert(colType.ItemType.IsBool);
return CreateArrayGetterDelegate<bool, DvBool>(index, x => x);
}
else if (outputType.GetElementType() == typeof(bool?))
{
Ch.Assert(colType.ItemType.IsBool);
return CreateArrayGetterDelegate<bool?, DvBool>(index, x => x ?? DvBool.NA);
}

// T[] -> VBuffer<T>
Ch.Assert(outputType.GetElementType() == colType.ItemType.RawType);
del = CreateArrayToVBufferGetter<int>;
if (outputType.GetElementType().IsGenericType && outputType.GetElementType().GetGenericTypeDefinition() == typeof(Nullable<>))
Ch.Assert(Nullable.GetUnderlyingType(outputType.GetElementType()) == colType.ItemType.RawType);
else
Ch.Assert(outputType.GetElementType() == colType.ItemType.RawType);
del = CreateDirectArrayGetterDelegate<int>;
genericType = outputType.GetElementType();
}
else if (colType.IsVector)
{
// VBuffer<T> -> VBuffer<T>
// REVIEW: Do we care about accomodating VBuffer<string> -> VBuffer<DvText>?
// REVIEW: why it's int and not long?

@Zruty0 Zruty0 Jul 19, 2018 •
edited
Loading

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

// REVIEW: why it's int and not long? [](start = 24, length = 37)

This is a strange place to add this comment.
We are doing MarshalInvoke here, right. We have the 'del' as a generic on some token type (it was historically always int), then in line 283 we make the correct getter. #Closed

Ch.Assert(outputType.IsGenericType);
Ch.Assert(outputType.GetGenericTypeDefinition() == typeof(VBuffer<>));
Ch.Assert(outputType.GetGenericArguments()[0] == colType.ItemType.RawType);
del = CreateVBufferToVBufferDelegate<int>;
del = CreateDirectVBufferGetterDelegate<int>;
}
else if (colType.IsPrimitive)
{
if (outputType == typeof(string))
{
// String -> DvText
Ch.Assert(colType.IsText);
return CreateStringToTextGetter(index);
return CreateGetterDelegate<String, DvText>(index, (x) => x == null ? DvText.NA : new DvText(x));
}
else if (outputType == typeof(bool))
{
// Bool -> DvBool
Ch.Assert(colType.IsBool);
return CreateBooleanToDvBoolGetter(index);
return CreateGetterDelegate<bool, DvBool>(index, x => x);
}
else if (outputType == typeof(bool?))
{
// Bool? -> DvBool
Ch.Assert(colType.IsBool);
return CreateNullableBooleanToDvBoolGetter(index);
return CreateGetterDelegate<bool?, DvBool>(index, x => x ?? DvBool.NA);
}
else if (outputType == typeof(int))
{
// int -> DvInt4
Ch.Assert(colType == NumberType.I4);
return CreateGetterDelegate<int, DvInt4>(index, x => x);
}
else if (outputType == typeof(int?))
{
// int -> DvInt4

@Zruty0 Zruty0 Jul 19, 2018 •
edited
Loading

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

int [](start = 31, length = 3)

int? #Closed

Ch.Assert(colType == NumberType.I4);
return CreateGetterDelegate<int?, DvInt4>(index, x => x ?? DvInt4.NA);
}
else if (outputType == typeof(short))
{
// short -> DvInt2
Ch.Assert(colType == NumberType.I2);
return CreateGetterDelegate<short, DvInt2>(index, x => x);
}
else if (outputType == typeof(short?))
{
// short? -> DvInt2
Ch.Assert(colType == NumberType.I2);
return CreateGetterDelegate<short?, DvInt2>(index, x => x ?? DvInt2.NA);
}
else if (outputType == typeof(long))
{
// long -> DvInt8
Ch.Assert(colType == NumberType.I8);
return CreateGetterDelegate<long, DvInt8>(index, x => x);
}
else if (outputType == typeof(long?))
{
// long? -> DvInt8
Ch.Assert(colType == NumberType.I8);
return CreateGetterDelegate<long?, DvInt8>(index, (x) => x.HasValue ? x.Value : DvInt8.NA);

@Zruty0 Zruty0 Jul 19, 2018 •
edited
Loading

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(x [](start = 78, length = 2)

why not x => x ?? DvInt8.NA here and below? #Closed

}
else if (outputType == typeof(sbyte))
{
// sbyte -> DvInt1
Ch.Assert(colType == NumberType.I1);
return CreateGetterDelegate<sbyte, DvInt1>(index, (x) => (DvInt1)x);
}
else if (outputType == typeof(sbyte?))
{
// sbyte -> DvInt1

@Zruty0 Zruty0 Jul 19, 2018 •
edited
Loading

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sbyte [](start = 31, length = 5)

sbyte? #Closed

Ch.Assert(colType == NumberType.I1);
return CreateGetterDelegate<sbyte?, DvInt1>(index, (x) => x.HasValue ? x.Value : DvInt1.NA);
}

// T -> T
Ch.Assert(colType.RawType == outputType);
del = CreateDirectGetter<int>;
if (outputType.IsGenericType && outputType.GetGenericTypeDefinition() == typeof(Nullable<>))
Ch.Assert(colType.RawType == Nullable.GetUnderlyingType(outputType));
else
Ch.Assert(colType.RawType == outputType);
del = CreateDirectGetterDelegate<int>;
}
else
{
// REVIEW: Is this even possible?
throw Ch.ExceptNotImpl("Type '{0}' is not yet supported.", outputType.FullName);
}
MethodInfo meth =
del.GetMethodInfo().GetGenericMethodDefinition().MakeGenericMethod(colType.ItemType.RawType);
del.GetMethodInfo().GetGenericMethodDefinition().MakeGenericMethod(genericType);
return (Delegate)meth.Invoke(this, new object[] { index });
}

private Delegate CreateStringArrayToVBufferGetter(int index)
private Delegate CreateArrayGetterDelegate<TSrc, TDst>(int index, Func<TSrc, TDst> convert)
{
var peek = DataView._peeks[index] as Peek<TRow, string[]>;
var peek = DataView._peeks[index] as Peek<TRow, TSrc[]>;
Ch.AssertValue(peek);

string[] buf = null;

return (ValueGetter<VBuffer<DvText>>)((ref VBuffer<DvText> dst) =>
TSrc[] buf = default;
return (ValueGetter<VBuffer<TDst>>)((ref VBuffer<TDst> dst) =>
{
peek(GetCurrentRowObject(), Position, ref buf);
var n = Utils.Size(buf);
dst = new VBuffer<DvText>(n, Utils.Size(dst.Values) < n
? new DvText[n]
dst = new VBuffer<TDst>(n, Utils.Size(dst.Values) < n
? new TDst[n]
: dst.Values, dst.Indices);
for (int i = 0; i < n; i++)
dst.Values[i] = new DvText(buf[i]);
});
}

private Delegate CreateStringToTextGetter(int index)
{
var peek = DataView._peeks[index] as Peek<TRow, string>;
Ch.AssertValue(peek);
string buf = null;
return (ValueGetter<DvText>)((ref DvText dst) =>
{
peek(GetCurrentRowObject(), Position, ref buf);
dst = new DvText(buf);
});
}

private Delegate CreateBooleanToDvBoolGetter(int index)
{
var peek = DataView._peeks[index] as Peek<TRow, bool>;
Ch.AssertValue(peek);
bool buf = false;
return (ValueGetter<DvBool>)((ref DvBool dst) =>
{
peek(GetCurrentRowObject(), Position, ref buf);
dst = (DvBool)buf;
dst.Values[i] = convert(buf[i]);
});
}

private Delegate CreateNullableBooleanToDvBoolGetter(int index)
private Delegate CreateGetterDelegate<TSrc, TDst>(int index, Func<TSrc, TDst> convert)
{
var peek = DataView._peeks[index] as Peek<TRow, bool?>;
var peek = DataView._peeks[index] as Peek<TRow, TSrc>;
Ch.AssertValue(peek);
bool? buf = null;
return (ValueGetter<DvBool>)((ref DvBool dst) =>
TSrc buf = default;
return (ValueGetter<TDst>)((ref TDst dst) =>
{
peek(GetCurrentRowObject(), Position, ref buf);
dst = buf.HasValue ? (DvBool)buf.Value : DvBool.NA;
dst = convert(buf);

@Zruty0 Zruty0 Jul 19, 2018 •
edited
Loading

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

convert [](start = 30, length = 7)

If convert is an identity, we are still incurring a delegate method call here, which is not efficient. #Closed

@Ivanidzo4ka Ivanidzo4ka Jul 19, 2018

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't have identity converts. even if I pass is x=>x I convert from TSrc to TDst class and call implicit converter. Which I'm not sure how to express through generic classes.
I would like to get rid of them, I just don't know way how to do it.

In reply to: 203875078 [](ancestors = 203875078)

@Zruty0 Zruty0 Jul 19, 2018

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh I see now. Ok, why don't we call this CreateConvertingGetterDelegate, to make it clear that there's also CreateDirectGetterDelegate?

In reply to: 203876308 [](ancestors = 203876308,203875078)

@Zruty0 Zruty0 Jul 19, 2018

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I seem to recall this conversation with Shon, that any 'helper' mechanism that would introduce these expensive conversions better doesn't exist at all.
I don't agree with it, but it's something to consider: with all but no warning to the user, the pipeline with int will be slower than the pipeline with float because of the conversion involved.

This is part of a conversation about why we even have DvInts.

Maybe a REVIEW comment alongside this converting getter/setter would be in order?

REVIEW: the converting getter invokes a type conversion delegate on every call, so it's inherently slower than the 'direct' getter. We don't have good indication of this to the user, and the selection of affected types is pretty arbitrary (signed integers and bools, but not uints and floats).

In reply to: 203877031 [](ancestors = 203877031,203876308,203875078)

});
}

private Delegate CreateArrayToVBufferGetter<TDst>(int index)
private Delegate CreateDirectArrayGetterDelegate<TDst>(int index)
{
var peek = DataView._peeks[index] as Peek<TRow, TDst[]>;
Ch.AssertValue(peek);
Expand All @@ -250,26 +330,29 @@ private Delegate CreateArrayToVBufferGetter<TDst>(int index)
});
}

private Delegate CreateVBufferToVBufferDelegate<TDst>(int index)
private Delegate CreateDirectVBufferGetterDelegate<TDst>(int index)
{
var peek = DataView._peeks[index] as Peek<TRow, VBuffer<TDst>>;
Ch.AssertValue(peek);
VBuffer<TDst> buf = default(VBuffer<TDst>);
return (ValueGetter<VBuffer<TDst>>)((ref VBuffer<TDst> dst) =>
{
// The peek for a VBuffer is just a simple assignment, so there is
// no copy going on in the peek, so we must do that as a second
// step to the destination.
peek(GetCurrentRowObject(), Position, ref buf);
buf.CopyTo(ref dst);
});
{
// The peek for a VBuffer is just a simple assignment, so there is
// no copy going on in the peek, so we must do that as a second
// step to the destination.
peek(GetCurrentRowObject(), Position, ref buf);
buf.CopyTo(ref dst);
});
}

private Delegate CreateDirectGetter<TDst>(int index)
private Delegate CreateDirectGetterDelegate<TDst>(int index)
{
var peek = DataView._peeks[index] as Peek<TRow, TDst>;
Ch.AssertValue(peek);
return (ValueGetter<TDst>)((ref TDst dst) => { peek(GetCurrentRowObject(), Position, ref dst); });
return (ValueGetter<TDst>)((ref TDst dst) =>
{
peek(GetCurrentRowObject(), Position, ref dst);
});
}

protected abstract TRow GetCurrentRowObject();
Expand Down
3 changes: 2 additions & 1 deletion src/Microsoft.ML.Api/SchemaDefinition.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -327,7 +327,8 @@ public static SchemaDefinition Create(Type userType)
// This field does not need a column.
// REVIEW: maybe validate the channel attribute now, instead
// of later at cursor creation.
if (fieldInfo.FieldType == typeof(IChannel))
// Const fields not need to be mapped.

@Zruty0 Zruty0 Jul 19, 2018 •
edited
Loading

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not need [](start = 31, length = 9)

do not need #Closed

@Zruty0 Zruty0 Jul 19, 2018

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, this comment only applies to the IsLiteral clause. I suggest to decouple the 2 clauses and have a different comment for each.

// Const fields not need to be mapped.
if (fieldInfo.IsLiteral)
                    continue;
// Clause to handle the field that may be used to expose the cursor channel. 
// This field does not need a column.
// REVIEW: maybe validate the channel attribute now, instead 
// of later at cursor creation.
if (fieldInfo.FieldType == typeof(IChannel) || fieldInfo.IsLiteral)
                    continue;

In reply to: 203871821 [](ancestors = 203871821)

if (fieldInfo.FieldType == typeof(IChannel) || fieldInfo.IsLiteral)
continue;

if (fieldInfo.GetCustomAttribute<NoColumnAttribute>() != null)
Expand Down
Loading