Skip to content

Fix column purpose for PipelineSweeperMacro #461

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
18 commits merged into from Jul 13, 2018
Merged

Fix column purpose for PipelineSweeperMacro #461

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
e1fa741
Merge pull request #1 from dotnet/master
abgoswam Jun 20, 2018
703cb38
Merge pull request #2 from dotnet/master
abgoswam Jun 27, 2018
c3c2125
Merge pull request #3 from dotnet/master
abgoswam Jun 28, 2018
17026a4
Adding arguments to PipelineSweep Macro
abgoswam Jun 29, 2018
c76181c
updating the unit tests
abgoswam Jun 30, 2018
28371fe
taking care of review comments; adding validations for Label, Weight,…
abgoswam Jul 1, 2018
310b395
taking care of some review comments
abgoswam Jul 2, 2018
4fbaf14
some code cleanup
abgoswam Jul 2, 2018
3ca4da9
addressing PR comments
abgoswam Jul 3, 2018
678d727
Merge pull request #4 from dotnet/master
abgoswam Jul 11, 2018
7f94a87
Merge pull request #5 from agoswamiazureml/master
abgoswam Jul 11, 2018
d55b1e0
API changes to use RoleMappedData
abgoswam Jul 11, 2018
c624415
Merge branch 'agoswami/purpose_inference' of https://github.com/agosw…
abgoswam Jul 11, 2018
d4f6102
taking care of review comments
abgoswam Jul 12, 2018
99172e2
using pipeline.UniqueId
abgoswam Jul 12, 2018
fb5ca79
taking care of review comments. update ColumnPurpose 'Group' so it is…
abgoswam Jul 12, 2018
6b34d71
Merge pull request #6 from dotnet/master
abgoswam Jul 12, 2018
3ad0c08
Merge pull request #7 from agoswamiazureml/master
abgoswam Jul 12, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 25 additions & 3 deletions src/Microsoft.ML.PipelineInference/AutoInference.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,9 +205,11 @@ public sealed class AutoMlMlState : IMlState
private IDataView _transformedData;
private ITerminator _terminator;
private string[] _requestedLearners;
private int _pipelineId;
private TransformInference.SuggestedTransform[] _availableTransforms;
private RecipeInference.SuggestedRecipe.SuggestedLearner[] _availableLearners;
private DependencyMap _dependencyMapping;
private Dictionary<string, ColumnPurpose> _columnPurpose;
public IPipelineOptimizer AutoMlEngine { get; set; }
public PipelinePattern[] BatchCandidates { get; set; }
public SupportedMetric Metric { get; }
Expand Down Expand Up @@ -370,19 +372,21 @@ private TransformInference.SuggestedTransform[] InferAndFilter(IDataView data, T
TransformInference.SuggestedTransform[] existingTransforms = null)
{
// Infer transforms using experts
var levelTransforms = TransformInference.InferTransforms(_env, data, args);
var levelTransforms = TransformInference.InferTransforms(_env, data, args, _columnPurpose);

// Retain only those transforms inferred which were also passed in.
if (existingTransforms != null)
return levelTransforms.Where(t => existingTransforms.Any(t2 => t2.Equals(t))).ToArray();
return levelTransforms;
}

public void InferSearchSpace(int numTransformLevels)
public void InferSearchSpace(int numTransformLevels, Dictionary<string, ColumnPurpose> columnPurpose = null)
{
var learners = RecipeInference.AllowedLearners(_env, TrainerKind).ToArray();
if (_requestedLearners != null && _requestedLearners.Length > 0)
learners = learners.Where(l => _requestedLearners.Contains(l.LearnerName)).ToArray();

_columnPurpose = columnPurpose;
ComputeSearchSpace(numTransformLevels, learners, (b, c) => InferAndFilter(b, c));
}

Expand Down Expand Up @@ -536,7 +540,25 @@ public PipelinePattern[] GetNextCandidates(int numberOfCandidates)
var currentBatchSize = numberOfCandidates;
if (_terminator is IterationTerminator itr)
currentBatchSize = Math.Min(itr.RemainingIterations(_history), numberOfCandidates);
BatchCandidates = AutoMlEngine.GetNextCandidates(_sortedSampledElements.Select(kvp => kvp.Value), currentBatchSize);
BatchCandidates = AutoMlEngine.GetNextCandidates(
_sortedSampledElements.Select(kvp => kvp.Value),
currentBatchSize,
_columnPurpose);

using (var ch = _host.Start("Suggested Pipeline"))
{
foreach (var pipeline in BatchCandidates)
{
ch.Info($"AutoInference Pipeline : {_pipelineId++}");
int transformK = 0;
foreach (var transform in pipeline.Transforms)
{
ch.Info($"AutoInference Transform {transformK++} : {transform.Transform}");
}
ch.Info($"AutoInference Learner : {pipeline.Learner}");
}
}

return BatchCandidates;
}

Expand Down
9 changes: 6 additions & 3 deletions src/Microsoft.ML.PipelineInference/AutoMlEngines/DefaultsEngine.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,11 @@ public DefaultsEngine(IHostEnvironment env, Arguments args)
_currentLearnerIndex = 0;
}

public override PipelinePattern[] GetNextCandidates(IEnumerable<PipelinePattern> history, int numCandidates)
public override PipelinePattern[] GetNextCandidates(IEnumerable<PipelinePattern> history, int numCandidates,
Dictionary<string, ColumnPurpose> colPurpose = null)
{
var candidates = new List<PipelinePattern>();
columnPurpose = colPurpose;

while (candidates.Count < numCandidates)
{
Expand All @@ -53,7 +55,8 @@ public override PipelinePattern[] GetNextCandidates(IEnumerable<PipelinePattern>

do
{ // Make sure transforms set is valid. Repeat until passes verifier.
pipeline = new PipelinePattern(SampleTransforms(out var transformsBitMask), learner, "", Env);
pipeline = new PipelinePattern(SampleTransforms(out var transformsBitMask),
learner, "", Env);
valid = PipelineVerifier(pipeline, transformsBitMask);
count++;
} while (!valid && count <= 1000);
Expand All @@ -77,7 +80,7 @@ private TransformInference.SuggestedTransform[] SampleTransforms(out long transf

// Add final features concat transform.
sampledTransforms.AddRange(AutoMlUtils.GetFinalFeatureConcat(Env, FullyTransformedData,
DependencyMapping, sampledTransforms.ToArray(), AvailableTransforms));
DependencyMapping, sampledTransforms.ToArray(), AvailableTransforms, columnPurpose));

return sampledTransforms.ToArray();
}
Expand Down
21 changes: 13 additions & 8 deletions src/Microsoft.ML.PipelineInference/AutoMlEngines/RocketEngine.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ private TransformInference.SuggestedTransform[] SampleTransforms(RecipeInference
// cause an error in verification, since it isn't included in the original
// dependency mapping (i.e., its level isn't in the dictionary).
sampledTransforms.AddRange(AutoMlUtils.GetFinalFeatureConcat(Env, FullyTransformedData,
DependencyMapping, sampledTransforms.ToArray(), AvailableTransforms));
DependencyMapping, sampledTransforms.ToArray(), AvailableTransforms, columnPurpose));
transformsBitMask = mask;

return sampledTransforms.ToArray();
Expand All @@ -202,9 +202,11 @@ private RecipeInference.SuggestedRecipe.SuggestedLearner[] GetTopLearners(IEnume
.Select(t=>AvailableLearners[t.Index]).ToArray();
}

public override PipelinePattern[] GetNextCandidates(IEnumerable<PipelinePattern> history, int numCandidates)
public override PipelinePattern[] GetNextCandidates(IEnumerable<PipelinePattern> history, int numCandidates,
Dictionary<string, ColumnPurpose> colPurpose = null)
{
var prevCandidates = history.ToArray();
columnPurpose = colPurpose;

switch (_currentStage)
{
Expand All @@ -220,7 +222,7 @@ public override PipelinePattern[] GetNextCandidates(IEnumerable<PipelinePattern>
// number of candidates, using second stage logic.
UpdateLearners(GetTopLearners(prevCandidates));
_currentStage++;
return GetNextCandidates(prevCandidates, numCandidates);
return GetNextCandidates(prevCandidates, numCandidates, columnPurpose);
}
else
return GetInitialPipelines(prevCandidates, remainingNum);
Expand Down Expand Up @@ -252,9 +254,11 @@ public override PipelinePattern[] GetNextCandidates(IEnumerable<PipelinePattern>
}
}

private PipelinePattern[] GetInitialPipelines(IEnumerable<PipelinePattern> history, int numCandidates) =>
_secondaryEngines[_randomInit ? nameof(UniformRandomEngine) : nameof(DefaultsEngine)]
.GetNextCandidates(history, numCandidates);
private PipelinePattern[] GetInitialPipelines(IEnumerable<PipelinePattern> history, int numCandidates)
{
var engine = _secondaryEngines[_randomInit ? nameof(UniformRandomEngine) : nameof(DefaultsEngine)];
return engine.GetNextCandidates(history, numCandidates, columnPurpose);
}

private PipelinePattern[] NextCandidates(PipelinePattern[] history, int numCandidates,
bool defaultHyperParams = false, bool uniformRandomTransforms = false)
Expand Down Expand Up @@ -294,8 +298,9 @@ private PipelinePattern[] NextCandidates(PipelinePattern[] history, int numCandi
do
{ // Make sure transforms set is valid and have not seen pipeline before.
// Repeat until passes or runs out of chances.
pipeline = new PipelinePattern(SampleTransforms(learner, history,
out var transformsBitMask, uniformRandomTransforms), learner, "", Env);
pipeline = new PipelinePattern(
SampleTransforms(learner, history, out var transformsBitMask, uniformRandomTransforms),
learner, "", Env);
hashKey = GetHashKey(transformsBitMask, learner);
valid = PipelineVerifier(pipeline, transformsBitMask) && !VisitedPipelines.Contains(hashKey);
count++;
Expand Down
6 changes: 4 additions & 2 deletions src/Microsoft.ML.PipelineInference/AutoMlEngines/UniformRandomEngine.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,10 @@ public UniformRandomEngine(IHostEnvironment env)
: base(env, env.Register("UniformRandomEngine(AutoML)"))
{}

public override PipelinePattern[] GetNextCandidates(IEnumerable<PipelinePattern> history, int numberOfCandidates)
public override PipelinePattern[] GetNextCandidates(IEnumerable<PipelinePattern> history, int numberOfCandidates,
Dictionary<string, ColumnPurpose> colPurpose = null)

@TomFinley TomFinley Jul 3, 2018 •
edited by ghost
Loading

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

colPurpose [](start = 46, length = 10)

I am somewhat unenthusiastic about this, since it basically seems like role-mappings, except in reverse (which also makes it, incidentally, less capable). Is there any possibility of re-using that concept, rather than inventing another thing that acts more or less just like it? #Resolved

@ghost ghost Jul 3, 2018 •
edited by ghost
Loading

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems the 'recipe framework' added the concept of 'Purpose' previously. Not quite sure why the decision was made at that time to introduce Purpose as an additional concept, instead of re-using the concept of Roles and extending it.

<snip from InferenceUtil.cs>
public enum ColumnPurpose
{
Ignore = 0,
Name = 1,
Label = 2,
NumericFeature = 3,
CategoricalFeature = 4,
TextFeature = 5,
Weight = 6,
GroupId = 7,
ImagePath = 8
}

Is there a simple way to extend Roles to include purpose (e.g. 'TextFeature', 'Ignore') ? My understanding is that it would be a bigger change that warrants a separate design / PR. Let me know if that sounds reasonable.

This PR is to address the particular case where 'recipe framework' inferred the purpose incorrectly. In sich cases the user has no way of overriding the inferred purpose when using the PipelineSweeperMacro. This PR unblocks such scenarios.

In reply to: 199661532 [](ancestors = 199661532)

@TomFinley TomFinley Jul 5, 2018 •
edited by ghost
Loading

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You don't extend roles, you simply use them. If you want to introduce roles appropriate for your task, that is totally fine and supported. I wonder though if the issue is that you do not have an actual ISchema implementation we're working on? #Resolved

@sfilipi sfilipi Jul 5, 2018 •
edited
Loading

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is ColumnPurpose and Roles the same things? Feels to me they are not. There is a ColumnPurpose of "ImagePath" but not such a Role, and @justin has ideas about adding more purposes. Purposes would be used to just communicate to Experts what transforms to suggest for the pipelines. Learners would not neccessarly need to know about them.

I have been wondering whether it makes more sense to extend the Loader columns to have a field/property of type ColumnPurpose, rather than pass yet another list of purposes, so they become part of the Schema, and whatever needs the purposes (the inference code) can just look it up in the columns.

In reply to: 200406010 [](ancestors = 200406010)

@ghost ghost Jul 6, 2018 •
edited by ghost
Loading

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

RecipeInference uses ColumnPurpose to infer the set of transforms to use (and also the columns used for each transform) .

In the GUI users can modify the ColumnPurpose through the wizard. We are adding arguments to the Macro so users can specify the purpose when using the macro programmatically.

Am not sure which ISchema implementation is being referred to. If you are referring to an ISchema implementation at the end of all transformations then yes, we do not have the final ISchema implemenation at this point in the computation. Hence the need to pass ColumnPurpose so we can infer the set of transforms and get to the final ISchema implementation.

Or, is the suggestion to completely do away with the notion of ColumnPurpose ?

In reply to: 200410413 [](ancestors = 200410413,200406010)

@ghost ghost Jul 12, 2018

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Made a couple of changes to address this:

  • Dropped ‘Ignore’ from Arguments to the PipelineSweepMacro.
  • Using RoleMappedData in the API (not exposing ColumnPurpose anymore in the APIs)

In reply to: 200723741 [](ancestors = 200723741,200410413,200406010)

{
columnPurpose = colPurpose;
return GetRandomPipelines(numberOfCandidates);
}

Expand Down Expand Up @@ -66,7 +68,7 @@ private PipelinePattern[] GetRandomPipelines(int numOfPipelines)

// Always include features concat transform
selectedTransforms.AddRange(AutoMlUtils.GetFinalFeatureConcat(Env, FullyTransformedData,
DependencyMapping, selectedTransforms.ToArray(), AvailableTransforms));
DependencyMapping, selectedTransforms.ToArray(), AvailableTransforms, columnPurpose));

// Compute hash key for checking if we've already seen this pipeline.
// However, if we keep missing, don't want to get stuck in infinite loop.
Expand Down
10 changes: 6 additions & 4 deletions src/Microsoft.ML.PipelineInference/AutoMlUtils.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,8 @@ public static long TransformsToBitmask(TransformInference.SuggestedTransform[] t
/// (In other words, if there would be nothing for that concatenate transform to do.)
/// </summary>
private static TransformInference.SuggestedTransform[] GetFinalFeatureConcat(IHostEnvironment env,
IDataView dataSample, int[] excludedColumnIndices, int level, int atomicIdOffset)
IDataView dataSample, int[] excludedColumnIndices, int level, int atomicIdOffset,
Dictionary<string, ColumnPurpose> columnPurpose = null)
{
var finalArgs = new TransformInference.Arguments
{
Expand All @@ -266,7 +267,7 @@ private static TransformInference.SuggestedTransform[] GetFinalFeatureConcat(IHo
ExcludedColumnIndices = excludedColumnIndices
};

var featuresConcatTransforms = TransformInference.InferConcatNumericFeatures(env, dataSample, finalArgs);
var featuresConcatTransforms = TransformInference.InferConcatNumericFeatures(env, dataSample, finalArgs, columnPurpose);

for (int i = 0; i < featuresConcatTransforms.Length; i++)
{
Expand All @@ -282,7 +283,8 @@ private static TransformInference.SuggestedTransform[] GetFinalFeatureConcat(IHo
/// </summary>
public static TransformInference.SuggestedTransform[] GetFinalFeatureConcat(IHostEnvironment env, IDataView data,
AutoInference.DependencyMap dependencyMapping, TransformInference.SuggestedTransform[] selectedTransforms,
TransformInference.SuggestedTransform[] allTransforms)
TransformInference.SuggestedTransform[] allTransforms,
Dictionary<string, ColumnPurpose> columnPurpose = null)
{
int level = 1;
int atomicGroupLimit = 0;
Expand All @@ -292,7 +294,7 @@ public static TransformInference.SuggestedTransform[] GetFinalFeatureConcat(IHos
atomicGroupLimit = allTransforms.Max(t => t.AtomicGroupId) + 1;
}
var excludedColumnIndices = GetExcludedColumnIndices(selectedTransforms, data, dependencyMapping);
return GetFinalFeatureConcat(env, data, excludedColumnIndices, level, atomicGroupLimit);
return GetFinalFeatureConcat(env, data, excludedColumnIndices, level, atomicGroupLimit, columnPurpose);
}

public static IDataView ApplyTransformSet(IHostEnvironment env, IDataView data, TransformInference.SuggestedTransform[] transforms)
Expand Down
7 changes: 5 additions & 2 deletions src/Microsoft.ML.PipelineInference/Interfaces/IPipelineOptimizer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ namespace Microsoft.ML.Runtime.PipelineInference
/// </summary>
public interface IPipelineOptimizer
{
PipelinePattern[] GetNextCandidates(IEnumerable<PipelinePattern> history, int numberOfCandidates);
PipelinePattern[] GetNextCandidates(IEnumerable<PipelinePattern> history, int numberOfCandidates,
Dictionary<string, ColumnPurpose> columnPurpose = null);

void SetSpace(TransformInference.SuggestedTransform[] availableTransforms,
RecipeInference.SuggestedRecipe.SuggestedLearner[] availableLearners,
Expand All @@ -44,6 +45,7 @@ public abstract class PipelineOptimizerBase : IPipelineOptimizer
protected IDataView OriginalData;
protected IDataView FullyTransformedData;
protected AutoInference.DependencyMap DependencyMapping;
protected Dictionary<string, ColumnPurpose> columnPurpose;

@TomFinley TomFinley Jul 3, 2018 •
edited by ghost
Loading

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

columnPurpose; [](start = 52, length = 14)

C# naming conventions, please. #Resolved

@ghost ghost Jul 3, 2018

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed.

In reply to: 199662160 [](ancestors = 199662160)

protected readonly IHostEnvironment Env;
protected readonly IHost Host;
protected readonly Dictionary<long, bool> TransformsMaskValidity;
Expand All @@ -60,7 +62,8 @@ protected PipelineOptimizerBase(IHostEnvironment env, IHost host)
ProbUtils = new SweeperProbabilityUtils(host);
}

public abstract PipelinePattern[] GetNextCandidates(IEnumerable<PipelinePattern> history, int numberOfCandidates);
public abstract PipelinePattern[] GetNextCandidates(IEnumerable<PipelinePattern> history, int numberOfCandidates,
Dictionary<string, ColumnPurpose> columnPurpose = null);

public virtual void SetSpace(TransformInference.SuggestedTransform[] availableTransforms,
RecipeInference.SuggestedRecipe.SuggestedLearner[] availableLearners,
Expand Down
Loading