Skip to content
This repository was archived by the owner on Nov 16, 2023. It is now read-only.

Update to ML.NET 0.10.preview #77

Merged
Shahab Moradi (shmoradims) merged 42 commits into from
Jan 16, 2019
Merged

Update to ML.NET 0.10.preview #77

Show file tree
Hide file tree
Changes from 41 commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
b59cbdb
Updating ML.NET nugets to latest 0.9 preview.
montebhoover Dec 20, 2018
4012a78
--generate_entrypoints phase 1
Jan 3, 2019
7788922
Fixed Models.CrossValidator
Jan 4, 2019
af12201
Updated all entrypoints
Jan 4, 2019
29d3006
New manifest.json, picket from Monte's branch
Jan 4, 2019
7e05c53
Updated API codegen
Jan 7, 2019
dcd749d
Replace ISchema and SchemaImpl with Schema and SchemaBuilder.
montebhoover Jan 7, 2019
78aca37
Merge branch 'master' into mlnet0.9
montebhoover Jan 10, 2019
2b3aafe
Revert "Replace ISchema and SchemaImpl with Schema and SchemaBuilder."
montebhoover Jan 10, 2019
5890ea9
Refactor IRowCursor to RowCursor.
montebhoover Jan 10, 2019
ce50654
Update ML.NET version in build.csproj.
montebhoover Jan 10, 2019
0b0b9dd
Update manifest.json to ml.net commit 92e762686989215ddf45d9db3f0a1c9…
Jan 10, 2019
b9f153b
Updated RunGraph.cs to ml.net 0.10
Jan 10, 2019
20713de
Merge branch 'mlnet0.9' into mlnet0.9_ep
Jan 10, 2019
9dbca28
Refactor Vbuffer
montebhoover Jan 10, 2019
e43ba6d
Added override to RowCursor methods
montebhoover Jan 10, 2019
1fd7f36
Merge pull request #1 from shmoradims/mlnet0.9_ep
montebhoover Jan 10, 2019
83e3def
Merge pull request #2 from shmoradims/fix_graphrunner
montebhoover Jan 10, 2019
913b04b
Update to NimbusML-privileged nugets from ML.NET.
montebhoover Jan 10, 2019
c75319a
Merge branch 'mlnet0.9' of https://github.com/montebhoover/NimbusML i…
montebhoover Jan 10, 2019
6fdc1ae
Update to Microsoft.ML namespace without Runtime.
montebhoover Jan 11, 2019
0cda66e
Schema and VBuffer fixes in NativeDataInterop.
montebhoover Jan 11, 2019
66cb74f
API fixes for IRandom and IsText in RmlEnvironment and NativeDataView.
montebhoover Jan 11, 2019
85849ec
Work on getting VBuffer pointers from Spans.
montebhoover Jan 11, 2019
cae5813
Some VBuffer fixes
Jan 11, 2019
b85788b
Merge branch 'mlnet0.9' of https://github.com/montebhoover/NimbusML i…
Jan 11, 2019
b7848af
fix some class names
Jan 11, 2019
1b64023
Fix Register Assembly names.
montebhoover Jan 11, 2019
ef02813
Remove ML.PipelineInference
montebhoover Jan 11, 2019
82edcec
fixed more classes
Jan 11, 2019
e02491c
Merge conflict
Jan 11, 2019
fea8196
Add back columndropper for backward compatability.
montebhoover Jan 11, 2019
2390143
Merge branch 'mlnet0.9' of https://github.com/montebhoover/NimbusML i…
montebhoover Jan 11, 2019
9c04b7f
Register Entrypoints assembly in environment.
montebhoover Jan 11, 2019
829bcfd
Fix homebrew update problem on VS Hosted Mac images.
montebhoover Jan 11, 2019
8452453
Updated all the nuget versions to be the same.
Jan 14, 2019
e9c6923
Merge branch 'mlnet0.9' of https://github.com/montebhoover/NimbusML i…
Jan 14, 2019
f9d5b71
Attempt to fix the dataframe unit tests
Jan 15, 2019
daba868
Fixed test_pyproj
Jan 15, 2019
053946a
Optimized VBuffer changes
Jan 15, 2019
4b97c77
Changed bridge version value to 0.10
Jan 15, 2019
47e44fe
Addressed PR comments
Jan 16, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion build/ci/phase-template.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ phases:
- script: $(_buildScript) --configuration $(_configuration) --runTests
# Mac phases
- ${{ if eq(parameters.name, 'Mac') }}:
- script: brew install libomp mono-libgdiplus gettext && brew link gettext --force
- script: brew update && brew install libomp mono-libgdiplus gettext && brew link gettext --force
- ${{ if eq(parameters.testDistro, 'noTests') }}:
- script: chmod 777 $(_buildScript) && $(_buildScript) --configuration $(_configuration)
- ${{ if eq(parameters.testDistro, '') }}:
Expand Down
38 changes: 19 additions & 19 deletions src/DotNetBridge/Bridge.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,19 +7,18 @@
using System.Runtime.InteropServices;
using System.Text;
using System.Threading;
using Microsoft.ML.Runtime;
using Microsoft.ML.Runtime.Data;
using Microsoft.ML.Runtime.ImageAnalytics;
using Microsoft.ML.Runtime.Learners;
using Microsoft.ML.Runtime.LightGBM;
using Microsoft.ML.Runtime.Model.Onnx;
using Microsoft.ML.Runtime.PipelineInference;
using Microsoft.ML;
using Microsoft.ML.Data;
using Microsoft.ML.EntryPoints;
using Microsoft.ML.ImageAnalytics;
using Microsoft.ML.LightGBM;
using Microsoft.ML.Model.Onnx;
using Microsoft.ML.Trainers;
using Microsoft.ML.Trainers.FastTree;
using Microsoft.ML.Trainers.KMeans;
using Microsoft.ML.Trainers.PCA;
using Microsoft.ML.Trainers.SymSgd;
using Microsoft.ML.Transforms;
using Microsoft.ML.Transforms.Categorical;

namespace Microsoft.MachineLearning.DotNetBridge
{
Expand Down Expand Up @@ -313,18 +312,19 @@ private static unsafe int GenericExec(EnvironmentBlock* penv, sbyte* psz, int cd
{
var host = env.Register("ML.NET_Execution");
env.ComponentCatalog.RegisterAssembly(typeof(TextLoader).Assembly); // ML.Data
env.ComponentCatalog.RegisterAssembly(typeof(LinearPredictor).Assembly); // ML.StandardLearners
env.ComponentCatalog.RegisterAssembly(typeof(CategoricalTransform).Assembly); // ML.Transforms
env.ComponentCatalog.RegisterAssembly(typeof(FastTreeBinaryPredictor).Assembly); // ML.FastTree
env.ComponentCatalog.RegisterAssembly(typeof(KMeansPredictor).Assembly); // ML.KMeansClustering
env.ComponentCatalog.RegisterAssembly(typeof(PcaPredictor).Assembly); // ML.PCA
env.ComponentCatalog.RegisterAssembly(typeof(Experiment).Assembly); // ML.Legacy
env.ComponentCatalog.RegisterAssembly(typeof(LightGbmBinaryPredictor).Assembly);
env.ComponentCatalog.RegisterAssembly(typeof(TensorFlowTransform).Assembly);
env.ComponentCatalog.RegisterAssembly(typeof(ImageLoaderTransform).Assembly);
env.ComponentCatalog.RegisterAssembly(typeof(StochasticGradientDescentClassificationTrainer).Assembly); // ML.StandardLearners
env.ComponentCatalog.RegisterAssembly(typeof(CategoricalCatalog).Assembly); // ML.Transforms
env.ComponentCatalog.RegisterAssembly(typeof(FastTreeRegressionTrainer).Assembly); // ML.FastTree
env.ComponentCatalog.RegisterAssembly(typeof(KMeansPlusPlusTrainer).Assembly); // ML.KMeansClustering
env.ComponentCatalog.RegisterAssembly(typeof(RandomizedPcaTrainer).Assembly); // ML.PCA
//env.ComponentCatalog.RegisterAssembly(typeof(Experiment).Assembly); // ML.Legacy
env.ComponentCatalog.RegisterAssembly(typeof(LightGbmRegressorTrainer).Assembly);
env.ComponentCatalog.RegisterAssembly(typeof(TensorFlowTransformer).Assembly);
env.ComponentCatalog.RegisterAssembly(typeof(ImageLoaderTransformer).Assembly);
env.ComponentCatalog.RegisterAssembly(typeof(SymSgdClassificationTrainer).Assembly);
env.ComponentCatalog.RegisterAssembly(typeof(AutoInference).Assembly);
env.ComponentCatalog.RegisterAssembly(typeof(SaveOnnxCommand).Assembly);
//env.ComponentCatalog.RegisterAssembly(typeof(AutoInference).Assembly); // ML.PipelineInference
env.ComponentCatalog.RegisterAssembly(typeof(OnnxExportExtensions).Assembly); // ML.Onnx
env.ComponentCatalog.RegisterAssembly(typeof(DataViewReference).Assembly);
//env.ComponentCatalog.RegisterAssembly(typeof(EnsemblePredictor).Assembly); // // ML.Ensemble BUG https://github.com/dotnet/machinelearning/issues/1078 Ensemble isn't in a NuGet package

using (var ch = host.Start("Executing"))
Expand Down
17 changes: 9 additions & 8 deletions src/DotNetBridge/DotNetBridge.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
<AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
<OutputPath>..\$(Platform)\$(Configuration)\</OutputPath>
<Configurations>DbgWinPy3.6;DbgWinPy3.5;DbgWinPy2.7;RlsWinPy3.6;RlsWinPy3.5;RlsWinPy2.7;DbgLinPy3.6;DbgLinPy3.5;DbgLinPy2.7;RlsLinPy3.6;RlsLinPy3.5;RlsLinPy2.7;RlsMacPy3.6</Configurations>
<Version>0.6.0</Version>
<Version>0.10.0</Version>
<Company>Microsoft Corporation</Company>
<Copyright>(c) Microsoft Corporation. All rights reserved.</Copyright>
<RepositoryUrl>https://github.com/Microsoft/NimbusML</RepositoryUrl>
Expand All @@ -31,12 +31,13 @@
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.ML" Version="0.7.0" />
<PackageReference Include="Microsoft.ML.CpuMath" Version="0.7.0" />
<PackageReference Include="Microsoft.ML.HalLearners" Version="0.7.0" />
<PackageReference Include="Microsoft.ML.ImageAnalytics" Version="0.7.0" />
<PackageReference Include="Microsoft.ML.LightGBM" Version="0.7.0" />
<PackageReference Include="Microsoft.ML.Onnx" Version="0.7.0" />
<PackageReference Include="Microsoft.ML.TensorFlow" Version="0.7.0" />
<PackageReference Include="Microsoft.ML" Version="0.10.0-preview-27310-10" />

@yaeldekel Yael Dekel (yaeldekel) Jan 16, 2019 •
edited by shmoradims
Loading

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

0.10.0-preview-27310-10 [](start = 54, length = 23)

Is this intentional? Shouldn't we use a release version? #Resolved

@shmoradims Shahab Moradi (shmoradims) Jan 16, 2019

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. 0.9 won't work and we don't have the official 0.10 yet. So we're stuck with preview nugets until 0.10 is released.

In reply to: 248377040 [](ancestors = 248377040)

@shmoradims Shahab Moradi (shmoradims) Jan 16, 2019

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, we won't be publishing a new nimbusml wheel file until we use 0.10.

In reply to: 248401711 [](ancestors = 248401711,248377040)

<PackageReference Include="Microsoft.ML.CpuMath" Version="0.10.0-preview-27310-10" />
<PackageReference Include="Microsoft.ML.EntryPoints" Version="0.10.0-preview-27310-10" />
<PackageReference Include="Microsoft.ML.HalLearners" Version="0.10.0-preview-27310-10" />
<PackageReference Include="Microsoft.ML.ImageAnalytics" Version="0.10.0-preview-27310-10" />
<PackageReference Include="Microsoft.ML.LightGBM" Version="0.10.0-preview-27310-10" />
<PackageReference Include="Microsoft.ML.Onnx" Version="0.10.0-preview-27310-10" />
<PackageReference Include="Microsoft.ML.TensorFlow" Version="0.10.0-preview-27310-10" />
</ItemGroup>
</Project>
2 changes: 1 addition & 1 deletion src/DotNetBridge/MessageValidator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

using System;
using System.Globalization;
using Microsoft.ML.Runtime;
using Microsoft.ML;

namespace Microsoft.MachineLearning.DotNetBridge
{
Expand Down
48 changes: 24 additions & 24 deletions src/DotNetBridge/NativeDataInterop.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@
using System.Globalization;
using System.Runtime.InteropServices;
using System.Text;
using Microsoft.ML.Runtime;
using Microsoft.ML.Runtime.Data;
using Microsoft.ML;
using Microsoft.ML.Data;

namespace Microsoft.MachineLearning.DotNetBridge
{
Expand Down Expand Up @@ -115,14 +115,14 @@ private static unsafe void SendViewToNative(IChannel ch, EnvironmentBlock* penv,
var expandCols = new HashSet<int>();
var allNames = new HashSet<string>();

for (int col = 0; col < schema.ColumnCount; col++)
for (int col = 0; col < schema.Count; col++)
{
if (schema.IsHidden(col))
if (schema[col].IsHidden)
continue;

var fullType = schema.GetColumnType(col);
var fullType = schema[col].Type;
var itemType = fullType.ItemType;
var name = schema.GetColumnName(col);
var name = schema[col].Name;

DataKind kind = itemType.RawKind;
int keyCard;
Expand Down Expand Up @@ -157,10 +157,10 @@ private static unsafe void SendViewToNative(IChannel ch, EnvironmentBlock* penv,
}

keyCard = itemType.KeyCount;
if (!schema.HasKeyNames(col, keyCard))
if (!schema[col].HasKeyValues(keyCard))
keyCard = -1;
}
else if (itemType.IsStandardScalar)
else if (itemType.IsStandardScalar())
{
switch (itemType.RawKind)
{
Expand Down Expand Up @@ -201,10 +201,10 @@ private static unsafe void SendViewToNative(IChannel ch, EnvironmentBlock* penv,
for (int i = 0; i < nSlots; i++)
AddUniqueName(info.SlotNames[i], allNames, nameIndices, nameUtf8Bytes);
}
else if (schema.HasSlotNames(col, nSlots))
else if (schema[col].HasSlotNames(nSlots))
{
var romNames = default(VBuffer<ReadOnlyMemory<char>>);
schema.GetMetadata(MetadataUtils.Kinds.SlotNames, col, ref romNames);
schema[col].Metadata.GetValue(MetadataUtils.Kinds.SlotNames, ref romNames);
foreach (var kvp in romNames.Items(true))
{
// REVIEW: Add the proper number of zeros to the slot index to make them sort in the right order.
Expand Down Expand Up @@ -273,12 +273,12 @@ private static unsafe void SendViewToNative(IChannel ch, EnvironmentBlock* penv,
var keyIndex = 0;
for (int i = 0; i < colIndices.Count; i++)
{
var type = schema.GetColumnType(colIndices[i]);
if (type.ItemType.IsKey && schema.HasKeyNames(colIndices[i], type.ItemType.KeyCount))
var type = schema[colIndices[i]].Type;
if (type.ItemType.IsKey && schema[colIndices[i]].HasKeyValues(type.ItemType.KeyCount))
{
ch.Assert(schema.HasKeyNames(colIndices[i], type.ItemType.KeyCount));
ch.Assert(schema[colIndices[i]].HasKeyValues(type.ItemType.KeyCount));
var keyValues = default(VBuffer<ReadOnlyMemory<char>>);
schema.GetMetadata(MetadataUtils.Kinds.KeyValues, colIndices[i], ref keyValues);
schema[colIndices[i]].Metadata.GetValue(MetadataUtils.Kinds.KeyValues, ref keyValues);
for (int slot = 0; slot < type.ValueCount; slot++)
{
foreach (var kvp in keyValues.Items())
Expand Down Expand Up @@ -333,15 +333,15 @@ private abstract unsafe class BufferFillerBase
public delegate void ValuePoker<T>(T value, int col, long index);

protected readonly int _colIndex;
protected readonly IRow _input;
protected readonly Row _input;

protected BufferFillerBase(IRow input, int pyColIndex)
protected BufferFillerBase(Row input, int pyColIndex)
{
_colIndex = pyColIndex;
_input = input;
}

public static BufferFillerBase Create(EnvironmentBlock* penv, IRow input, int pyCol, int idvCol, DataKind dataKind, ColumnType type, void* setter)
public static BufferFillerBase Create(EnvironmentBlock* penv, Row input, int pyCol, int idvCol, DataKind dataKind, ColumnType type, void* setter)
{
var itemType = type.ItemType;
// We convert the unsigned types to signed types, with -1 indicating missing in Python.
Expand Down Expand Up @@ -494,14 +494,14 @@ private sealed class Impl<TSrc> : BufferFillerBase
private readonly ValueGetter<TSrc> _get;
private readonly ValuePoker<TSrc> _poker;

public Impl(IRow input, int pyColIndex, int idvColIndex, ColumnType type, ValuePoker<TSrc> poker)
public Impl(Row input, int pyColIndex, int idvColIndex, ColumnType type, ValuePoker<TSrc> poker)
: base(input, pyColIndex)
{
Contracts.AssertValue(input);
Contracts.Assert(0 <= idvColIndex && idvColIndex < input.Schema.ColumnCount);
Contracts.Assert(0 <= idvColIndex && idvColIndex < input.Schema.Count);

if (type.IsVector)
_getVec = RowCursorUtils.GetVecGetterAs<TSrc>(type.ItemType.AsPrimitive, input, idvColIndex);
_getVec = RowCursorUtils.GetVecGetterAs<TSrc>((PrimitiveType)type.ItemType, input, idvColIndex);
else
_get = RowCursorUtils.GetGetterAs<TSrc>(type, input, idvColIndex);

Expand All @@ -516,19 +516,19 @@ public override void Set()
{
for (int i = 0; i < _buffer.Length; i++)
{
_poker(_buffer.Values[i], _colIndex + i, _input.Position);
_poker(_buffer.GetValues()[i], _colIndex + i, _input.Position);
}
}
else
{
int ii = 0;
for (int i = 0; i < _buffer.Length; i++)
{
while (ii < _buffer.Count && _buffer.Indices[ii] < i)
while (ii < _buffer.GetValues().Length && _buffer.GetIndices()[ii] < i)

@yaeldekel Yael Dekel (yaeldekel) Jan 16, 2019 •
edited by shmoradims
Loading

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

GetValues [](start = 52, length = 9)

It may be cleaner to add two lines before the for loop:
var values=_buffer.GetValues();
var indices=_buffer.GetIndices();
#Resolved

@shmoradims Shahab Moradi (shmoradims) Jan 16, 2019

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Changed

In reply to: 248379904 [](ancestors = 248379904)

ii++;
TSrc val = default(TSrc);
if (ii < _buffer.Count && _buffer.Indices[ii] == i)
val = _buffer.Values[ii];
if (ii < _buffer.GetValues().Length && _buffer.GetIndices()[ii] == i)
val = _buffer.GetValues()[ii];
_poker(val, _colIndex + i, _input.Position);
}
}
Expand Down
Loading