Skip to content

add diskConvert option in fast tree search space #6316

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
May 11, 2023
Merged

add diskConvert option in fast tree search space #6316

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
778b905
update
LittleLittleCloud Sep 8, 2022
b50a10e
Update AutoMLExperimentTests.cs
LittleLittleCloud Sep 8, 2022
0cc2a2f
Merge branch 'main' into u/xiaoyun/330GB
LittleLittleCloud Oct 12, 2022
e601589
checkout test file
LittleLittleCloud May 10, 2023
d0c54a4
Merge branch 'main' into u/xiaoyun/330GB
LittleLittleCloud May 10, 2023
abad9ef
fix tests
LittleLittleCloud May 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ public virtual void ReportCompletedTrial(TrialResult result)

public virtual void ReportFailTrial(TrialSettings settings, Exception exception = null)
{
_logger.Trace(exception.Message + exception.StackTrace);
_logger.Info($"Update Failed Trial - Id: {settings.TrialId} - Pipeline: {_pipeline.ToString(settings.Parameter)}");
}

Expand Down
5 changes: 5 additions & 0 deletions src/Microsoft.ML.AutoML/CodeGen/fast_tree_search_space.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,11 @@
{
"name": "ExampleWeightColumnName",
"type": "string"
},
{
"name": "DiskTranspose",
Copy link
Member Author

@LittleLittleCloud LittleLittleCloud Sep 8, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This option allows user to overwrite DiskTranspose when necessary, which avoids OOM when dataset is large

"type": "boolean",
"default": false
}
]
}
1 change: 1 addition & 0 deletions src/Microsoft.ML.AutoML/CodeGen/search-space-schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,7 @@
"Quiet",
"OutputAsFloatArray",
"ModelFactory",
"DiskTranspose",
"Sentence1ColumnName",
"Sentence2ColumnName",
"BatchSize",
Expand Down
4 changes: 4 additions & 0 deletions src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/FastTree.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ public override IEstimator<ITransformer> BuildFromOption(MLContext context, Fast
NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(),
MaximumBinCountPerFeature = param.MaximumBinCountPerFeature,
FeatureFraction = param.FeatureFraction,
DiskTranspose = param.DiskTranspose,
};

return context.MulticlassClassification.Trainers.OneVersusAll(context.BinaryClassification.Trainers.FastTree(option), labelColumnName: param.LabelColumnName);
Expand All @@ -43,6 +44,7 @@ public override IEstimator<ITransformer> BuildFromOption(MLContext context, Fast
ExampleWeightColumnName = param.ExampleWeightColumnName,
NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(),
MaximumBinCountPerFeature = param.MaximumBinCountPerFeature,
DiskTranspose = param.DiskTranspose,
FeatureFraction = param.FeatureFraction,
};

Expand All @@ -65,6 +67,7 @@ public override IEstimator<ITransformer> BuildFromOption(MLContext context, Fast
ExampleWeightColumnName = param.ExampleWeightColumnName,
NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(),
MaximumBinCountPerFeature = param.MaximumBinCountPerFeature,
DiskTranspose = param.DiskTranspose,
FeatureFraction = param.FeatureFraction,
};

Expand All @@ -87,6 +90,7 @@ public override IEstimator<ITransformer> BuildFromOption(MLContext context, Fast
ExampleWeightColumnName = param.ExampleWeightColumnName,
NumberOfThreads = AutoMlUtils.GetNumberOfThreadFromEnvrionment(),
MaximumBinCountPerFeature = param.MaximumBinCountPerFeature,
DiskTranspose = param.DiskTranspose,
FeatureFraction = param.FeatureFraction,
};

Expand Down
3 changes: 2 additions & 1 deletion ...stimatorPipelineTest.SweepableEstimatorPipeline_search_space_init_value_test.approved.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
"FeatureFraction": 1,
"LearningRate": 0.10,
"LabelColumnName": "Label",
"FeatureColumnName": "Feature"
"FeatureColumnName": "Feature",
"DiskTranspose": false
}
}
3 changes: 2 additions & 1 deletion ...pableExtensionTest.CreateSweepablePipelineFromIEstimatorAndBinaryClassifiers.approved.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@
"FeatureFraction": 1,
"LearningRate": 0.1,
"LabelColumnName": "Label",
"FeatureColumnName": "Features"
"FeatureColumnName": "Features",
"DiskTranspose": false
}
},
"e2": {
Expand Down
3 changes: 2 additions & 1 deletion ...epableExtensionTest.CreateSweepablePipelineFromIEstimatorAndMultiClassifiers.approved.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@
"FeatureFraction": 1,
"LearningRate": 0.1,
"LabelColumnName": "Label",
"FeatureColumnName": "Features"
"FeatureColumnName": "Features",
"DiskTranspose": false
}
},
"e2": {
Expand Down
3 changes: 2 additions & 1 deletion ...ts/SweepableExtensionTest.CreateSweepablePipelineFromIEstimatorAndRegressors.approved.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@
"FeatureFraction": 1,
"LearningRate": 0.1,
"LabelColumnName": "Label",
"FeatureColumnName": "Features"
"FeatureColumnName": "Features",
"DiskTranspose": false
}
},
"e2": {
Expand Down
3 changes: 2 additions & 1 deletion ...xtensionTest.CreateSweepablePipelineFromIEstimatorAndSweepableEstimatorArray.approved.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@
"FeatureFraction": 1,
"LearningRate": 0.1,
"LabelColumnName": "Label",
"FeatureColumnName": "Features"
"FeatureColumnName": "Features",
"DiskTranspose": false
}
},
"e2": {
Expand Down
3 changes: 2 additions & 1 deletion ...tensionTest.CreateSweepablePipelineFromSweepableEstimatorAndMultiClassifiers.approved.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@
"FeatureFraction": 1,
"LearningRate": 0.1,
"LabelColumnName": "Label",
"FeatureColumnName": "Features"
"FeatureColumnName": "Features",
"DiskTranspose": false
}
},
"e2": {
Expand Down
3 changes: 2 additions & 1 deletion ...Test.CreateSweepablePipelineFromSweepableEstimatorAndSweepableEstimatorArray.approved.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@
"FeatureFraction": 1,
"LearningRate": 0.1,
"LabelColumnName": "Label",
"FeatureColumnName": "Features"
"FeatureColumnName": "Features",
"DiskTranspose": false
}
},
"e2": {
Expand Down
3 changes: 2 additions & 1 deletion ...xtensionTest.CreateSweepablePipelineFromSweepablePipelineAndMultiClassifiers.approved.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@
"FeatureFraction": 1,
"LearningRate": 0.1,
"LabelColumnName": "Label",
"FeatureColumnName": "Features"
"FeatureColumnName": "Features",
"DiskTranspose": false
}
},
"e3": {
Expand Down
3 changes: 2 additions & 1 deletion ...nTest.CreateSweepablePipelineFromSweepablePipelineAndSweepableEstimatorArray.approved.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@
"FeatureFraction": 1,
"LearningRate": 0.1,
"LabelColumnName": "Label",
"FeatureColumnName": "Features"
"FeatureColumnName": "Features",
"DiskTranspose": false
}
},
"e3": {
Expand Down