Skip to content

Statistics test updates for Arrow 6.0.0 #238

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jgiannuzzi merged 3 commits into from
Dec 2, 2021
Merged

Statistics test updates for Arrow 6.0.0 #238

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1a1897e
Re-enable null-count checks in statistics tests
adamreeve Dec 1, 2021
4f181cf
Re-enable min/max statistics tests for decimal
adamreeve Dec 1, 2021
575e597
Get decimal multiplier from column descriptor for min/max statistics …
adamreeve Dec 1, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 29 additions & 30 deletions csharp.test/TestLogicalTypeRoundtrip.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,17 +154,14 @@ private static void AssertReadRoundtrip(int rowsPerBatch, int readBufferLength,
if (expected.HasStatistics)
{
Assert.AreEqual(expected.HasMinMax, statistics?.HasMinMax);
//Assert.AreEqual(expected.NullCount, statistics?.NullCount);
//Assert.AreEqual(expected.NumValues, statistics?.NumValues);
Assert.AreEqual(expected.NullCount, statistics?.NullCount);
Assert.AreEqual(expected.NumValues, statistics?.NumValues);
Assert.AreEqual(expected.PhysicalType, statistics?.PhysicalType);

// BUG Don't check for decimal until https://issues.apache.org/jira/browse/ARROW-6149 is fixed.
var buggy = expected.LogicalType is DecimalLogicalType;

if (expected.HasMinMax && !buggy)
if (expected.HasMinMax)
{
Assert.AreEqual(expected.Min, expected.Converter(statistics!.MinUntyped));
Assert.AreEqual(expected.Max, expected.Converter(statistics!.MaxUntyped));
Assert.AreEqual(expected.Min, expected.Converter(statistics!.MinUntyped, descr));
Assert.AreEqual(expected.Max, expected.Converter(statistics!.MaxUntyped, descr));
}
}
else
Expand Down Expand Up @@ -648,7 +645,8 @@ private static ExpectedColumn[] CreateExpectedColumns()
Values = Enumerable.Range(0, NumRows).Select(i => ((decimal) i * i * i) / 1000 - 10).ToArray(),
Min = -10m,
Max = ((NumRows - 1m) * (NumRows - 1m) * (NumRows - 1m)) / 1000 - 10,
Converter = v => LogicalRead.ToDecimal((FixedLenByteArray) v, 3)
Converter = (v, descr) => LogicalRead.ToDecimal(
(FixedLenByteArray) v, Decimal128.GetScaleMultiplier(descr.TypeScale))
},
new ExpectedColumn
{
Expand All @@ -662,7 +660,8 @@ private static ExpectedColumn[] CreateExpectedColumns()
NumValues = NumRows - (NumRows + 10) / 11,
Min = -9.999m,
Max = ((NumRows - 1m) * (NumRows - 1m) * (NumRows - 1m)) / 1000 - 10,
Converter = v => LogicalRead.ToDecimal((FixedLenByteArray) v, 3)
Converter = (v, descr) => LogicalRead.ToDecimal(
(FixedLenByteArray) v, Decimal128.GetScaleMultiplier(descr.TypeScale))
},
new ExpectedColumn
{
Expand All @@ -674,7 +673,7 @@ private static ExpectedColumn[] CreateExpectedColumns()
Values = Enumerable.Range(0, NumRows).Select(i => new Guid(i, 0x1234, 0x5678, 0x9A, 0xBC, 0xDE, 0xF0, 0x12, 0x34, 0x56, 0x7F)).ToArray(),
Min = new Guid(0, 0x1234, 0x5678, 0x9A, 0xBC, 0xDE, 0xF0, 0x12, 0x34, 0x56, 0x7F),
Max = new Guid(NumRows - 1, 0x1234, 0x5678, 0x9A, 0xBC, 0xDE, 0xF0, 0x12, 0x34, 0x56, 0x7F),
Converter = v => LogicalRead.ToUuid((FixedLenByteArray) v)
Converter = (v, _) => LogicalRead.ToUuid((FixedLenByteArray) v)
},
new ExpectedColumn
{
Expand All @@ -688,7 +687,7 @@ private static ExpectedColumn[] CreateExpectedColumns()
NumValues = NumRows - (NumRows + 10) / 11,
Min = new Guid(1, 0x1234, 0x5678, 0x9A, 0xBC, 0xDE, 0xF0, 0x12, 0x34, 0x56, 0x7F),
Max = new Guid(NumRows - 1, 0x1234, 0x5678, 0x9A, 0xBC, 0xDE, 0xF0, 0x12, 0x34, 0x56, 0x7F),
Converter = v => LogicalRead.ToUuid((FixedLenByteArray) v)
Converter = (v, _) => LogicalRead.ToUuid((FixedLenByteArray) v)
},
new ExpectedColumn
{
Expand Down Expand Up @@ -718,7 +717,7 @@ private static ExpectedColumn[] CreateExpectedColumns()
Values = Enumerable.Range(0, NumRows).Select(i => new DateTime(2018, 01, 01) + TimeSpan.FromHours(i)).ToArray(),
Min = new DateTime(2018, 01, 01),
Max = new DateTime(2018, 01, 01) + TimeSpan.FromHours(NumRows - 1),
Converter = v => LogicalRead.ToDateTimeMicros((long) v)
Converter = (v, _) => LogicalRead.ToDateTimeMicros((long) v)
},
new ExpectedColumn
{
Expand All @@ -730,7 +729,7 @@ private static ExpectedColumn[] CreateExpectedColumns()
NumValues = NumRows - (NumRows + 10) / 11,
Min = new DateTime(2018, 01, 01) + TimeSpan.FromHours(1),
Max = new DateTime(2018, 01, 01) + TimeSpan.FromHours(NumRows - 1),
Converter = v => LogicalRead.ToDateTimeMicros((long) v)
Converter = (v, _) => LogicalRead.ToDateTimeMicros((long) v)
},
new ExpectedColumn
{
Expand All @@ -741,7 +740,7 @@ private static ExpectedColumn[] CreateExpectedColumns()
Values = Enumerable.Range(0, NumRows).Select(i => new DateTime(2018, 01, 01) + TimeSpan.FromHours(i)).ToArray(),
Min = new DateTime(2018, 01, 01),
Max = new DateTime(2018, 01, 01) + TimeSpan.FromHours(NumRows - 1),
Converter = v => LogicalRead.ToDateTimeMillis((long) v)
Converter = (v, _) => LogicalRead.ToDateTimeMillis((long) v)
},
new ExpectedColumn
{
Expand All @@ -754,7 +753,7 @@ private static ExpectedColumn[] CreateExpectedColumns()
NumValues = NumRows - (NumRows + 10) / 11,
Min = new DateTime(2018, 01, 01) + TimeSpan.FromHours(1),
Max = new DateTime(2018, 01, 01) + TimeSpan.FromHours(NumRows - 1),
Converter = v => LogicalRead.ToDateTimeMillis((long) v)
Converter = (v, _) => LogicalRead.ToDateTimeMillis((long) v)
},
new ExpectedColumn
{
Expand All @@ -764,7 +763,7 @@ private static ExpectedColumn[] CreateExpectedColumns()
Values = Enumerable.Range(0, NumRows).Select(i => new DateTimeNanos(new DateTime(2018, 01, 01) + TimeSpan.FromHours(i))).ToArray(),
Min = new DateTimeNanos(new DateTime(2018, 01, 01)),
Max = new DateTimeNanos(new DateTime(2018, 01, 01) + TimeSpan.FromHours(NumRows - 1)),
Converter = v => new DateTimeNanos((long) v)
Converter = (v, _) => new DateTimeNanos((long) v)
},
new ExpectedColumn
{
Expand All @@ -776,7 +775,7 @@ private static ExpectedColumn[] CreateExpectedColumns()
NumValues = NumRows - (NumRows + 10) / 11,
Min = new DateTimeNanos(new DateTime(2018, 01, 01) + TimeSpan.FromHours(1)),
Max = new DateTimeNanos(new DateTime(2018, 01, 01) + TimeSpan.FromHours(NumRows - 1)),
Converter = v => new DateTimeNanos((long) v)
Converter = (v, _) => new DateTimeNanos((long) v)
},
new ExpectedColumn
{
Expand All @@ -786,7 +785,7 @@ private static ExpectedColumn[] CreateExpectedColumns()
Values = Enumerable.Range(0, NumRows).Select(i => TimeSpan.FromHours(-13) + TimeSpan.FromHours(i)).ToArray(),
Min = TimeSpan.FromHours(-13),
Max = TimeSpan.FromHours(-13 + NumRows - 1),
Converter = v => LogicalRead.ToTimeSpanMicros((long) v)
Converter = (v, _) => LogicalRead.ToTimeSpanMicros((long) v)
},
new ExpectedColumn
{
Expand All @@ -798,7 +797,7 @@ private static ExpectedColumn[] CreateExpectedColumns()
NumValues = NumRows - (NumRows + 10) / 11,
Min = TimeSpan.FromHours(-13 + 1),
Max = TimeSpan.FromHours(-13 + NumRows - 1),
Converter = v => LogicalRead.ToTimeSpanMicros((long) v)
Converter = (v, _) => LogicalRead.ToTimeSpanMicros((long) v)
},
new ExpectedColumn
{
Expand All @@ -809,7 +808,7 @@ private static ExpectedColumn[] CreateExpectedColumns()
Values = Enumerable.Range(0, NumRows).Select(i => TimeSpan.FromHours(-13) + TimeSpan.FromHours(i)).ToArray(),
Min = TimeSpan.FromHours(-13),
Max = TimeSpan.FromHours(-13 + NumRows - 1),
Converter = v => LogicalRead.ToTimeSpanMillis((int) v)
Converter = (v, _) => LogicalRead.ToTimeSpanMillis((int) v)
},
new ExpectedColumn
{
Expand All @@ -822,7 +821,7 @@ private static ExpectedColumn[] CreateExpectedColumns()
NumValues = NumRows - (NumRows + 10) / 11,
Min = TimeSpan.FromHours(-13 + 1),
Max = TimeSpan.FromHours(-13 + NumRows - 1),
Converter = v => LogicalRead.ToTimeSpanMillis((int) v)
Converter = (v, _) => LogicalRead.ToTimeSpanMillis((int) v)
},
new ExpectedColumn
{
Expand All @@ -832,7 +831,7 @@ private static ExpectedColumn[] CreateExpectedColumns()
Values = Enumerable.Range(0, NumRows).Select(i => new TimeSpanNanos(TimeSpan.FromHours(-13) + TimeSpan.FromHours(i))).ToArray(),
Min = new TimeSpanNanos(TimeSpan.FromHours(-13)),
Max = new TimeSpanNanos(TimeSpan.FromHours(-13 + NumRows - 1)),
Converter = v => new TimeSpanNanos((long) v)
Converter = (v, _) => new TimeSpanNanos((long) v)
},
new ExpectedColumn
{
Expand All @@ -844,7 +843,7 @@ private static ExpectedColumn[] CreateExpectedColumns()
NumValues = NumRows - (NumRows + 10) / 11,
Min = new TimeSpanNanos(TimeSpan.FromHours(-13 + 1)),
Max = new TimeSpanNanos(TimeSpan.FromHours(-13 + NumRows - 1)),
Converter = v => new TimeSpanNanos((long) v)
Converter = (v, _) => new TimeSpanNanos((long) v)
},
new ExpectedColumn
{
Expand All @@ -856,7 +855,7 @@ private static ExpectedColumn[] CreateExpectedColumns()
NumValues = NumRows - (NumRows + 17) / 18,
Min = "",
Max = "Hello, 98!",
Converter = v => LogicalRead.ToString((ByteArray) v)
Converter = (v, _) => LogicalRead.ToString((ByteArray) v)
},
new ExpectedColumn
{
Expand All @@ -869,7 +868,7 @@ private static ExpectedColumn[] CreateExpectedColumns()
NumValues = NumRows - (NumRows + 8) / 9,
Min = "{ \"id\", 1 }",
Max = "{ \"id\", 98 }",
Converter = v => LogicalRead.ToString((ByteArray) v)
Converter = (v, _) => LogicalRead.ToString((ByteArray) v)
},
new ExpectedColumn
{
Expand All @@ -880,7 +879,7 @@ private static ExpectedColumn[] CreateExpectedColumns()
NumValues = NumRows - (NumRows + 5) / 6,
Min = new byte[0],
Max = BitConverter.GetBytes(NumRows - 1),
Converter = v => LogicalRead.ToByteArray((ByteArray) v)
Converter = (v, _) => LogicalRead.ToByteArray((ByteArray) v)
},
new ExpectedColumn
{
Expand All @@ -893,7 +892,7 @@ private static ExpectedColumn[] CreateExpectedColumns()
NumValues = NumRows - (NumRows + 2) / 3,
Min = BitConverter.GetBytes(1),
Max = BitConverter.GetBytes(NumRows - 1),
Converter = v => LogicalRead.ToByteArray((ByteArray) v)
Converter = (v, _) => LogicalRead.ToByteArray((ByteArray) v)
},
new ExpectedColumn
{
Expand Down Expand Up @@ -1006,7 +1005,7 @@ private static ExpectedColumn[] CreateExpectedColumns()
NumValues = (NumRows / 3 + 1) * 3,
Min = BitConverter.GetBytes(0),
Max = BitConverter.GetBytes(252),
Converter = v => LogicalRead.ToByteArray((ByteArray) v)
Converter = (v, _) => LogicalRead.ToByteArray((ByteArray) v)
}
};
}
Expand All @@ -1027,7 +1026,7 @@ private sealed class ExpectedColumn
public long NullCount;
public long NumValues = NumRows;

public Func<object, object> Converter = v => v;
public Func<object, ColumnDescriptor, object> Converter = (v, _) => v;
}

private const int NumRows = 119;
Expand Down