Skip to content

Reformatting TensorFlow and AnomalyDetection samples to width 85 #3922

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sayanshaw24 merged 9 commits into from
Jun 29, 2019
Merged

Reformatting TensorFlow and AnomalyDetection samples to width 85 #3922

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
94553e9
Just trying stuff out!
sayanshaw24 Jun 26, 2019
f24dd62
Testing
sayanshaw24 Jun 26, 2019
a0bc22b
Reformatted TensorFlow samples to width 85
sayanshaw24 Jun 27, 2019
cb39e85
Reformatted AnomalyDetection samples to width 85
sayanshaw24 Jun 27, 2019
8b218f0
Fixed formatting errors
sayanshaw24 Jun 27, 2019
6364b31
Merge branch 'master' of https://github.com/dotnet/machinelearning in…
sayanshaw24 Jun 27, 2019
d91086e
Made a few edits to TensorFlow samples format
sayanshaw24 Jun 28, 2019
65355fc
Fixed syntax errors
sayanshaw24 Jun 28, 2019
0fcebec
Spacing changes
sayanshaw24 Jun 29, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 21 additions & 10 deletions docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/ImageClassification.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@ public static void Example()
if (!File.Exists(modelLocation))
{
modelLocation = Download(@"https://storage.googleapis.com/download.tensorflow.org/models/tflite_11_05_08/resnet_v2_101.tgz", @"resnet_v2_101_299_frozen.tgz");
Unzip(Path.Join(Directory.GetCurrentDirectory(), modelLocation), Directory.GetCurrentDirectory());
Unzip(Path.Join(Directory.GetCurrentDirectory(), modelLocation),
Directory.GetCurrentDirectory());

modelLocation = "resnet_v2_101_299_frozen.pb";
}

Expand All @@ -32,7 +34,8 @@ public static void Example()
var idv = mlContext.Data.LoadFromEnumerable(data);

// Create a ML pipeline.
var pipeline = mlContext.Model.LoadTensorFlowModel(modelLocation).ScoreTensorFlowModel(
var pipeline = mlContext.Model.LoadTensorFlowModel(modelLocation)
.ScoreTensorFlowModel(
new[] { nameof(OutputScores.output) },
new[] { nameof(TensorData.input) }, addBatchDimensionInput: true);

Expand All @@ -41,15 +44,18 @@ public static void Example()
var transformedValues = estimator.Transform(idv);

// Retrieve model scores.
var outScores = mlContext.Data.CreateEnumerable<OutputScores>(transformedValues, reuseRowObject: false);
var outScores = mlContext.Data.CreateEnumerable<OutputScores>(
transformedValues, reuseRowObject: false);

// Display scores. (for the sake of brevity we display scores of the first 3 classes)
// Display scores. (for the sake of brevity we display scores of the
// first 3 classes)
foreach (var prediction in outScores)
{
int numClasses = 0;
foreach (var classScore in prediction.output.Take(3))
{
Console.WriteLine($"Class #{numClasses++} score = {classScore}");
Console.WriteLine(
$"Class #{numClasses++} score = {classScore}");
}
Console.WriteLine(new string('-', 10));
}
Expand All @@ -72,7 +78,8 @@ public static void Example()

/// <summary>
/// A class to hold sample tensor data.
/// Member name should match the inputs that the model expects (in this case, input).
/// Member name should match the inputs that the model expects (in this
/// case, input).
/// </summary>
public class TensorData
{
Expand All @@ -86,9 +93,12 @@ public class TensorData
public static TensorData[] GetTensorData()
{
// This can be any numerical data. Assume image pixel values.
var image1 = Enumerable.Range(0, inputSize).Select(x => (float)x / inputSize).ToArray();
var image2 = Enumerable.Range(0, inputSize).Select(x => (float)(x + 10000) / inputSize).ToArray();
return new TensorData[] { new TensorData() { input = image1 }, new TensorData() { input = image2 } };
var image1 = Enumerable.Range(0, inputSize).Select(
x => (float)x / inputSize).ToArray();
var image2 = Enumerable.Range(0, inputSize).Select(
x => (float)(x + 10000) / inputSize).ToArray();

@codemzs codemzs Jun 28, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As discussed new line between any two lines that were broken into multiple lines.

return new TensorData[] { new TensorData() { input = image1 },
new TensorData() { input = image2 } };
}

/// <summary>
Expand All @@ -110,7 +120,8 @@ private static string Download(string baseGitPath, string dataFile)
}

/// <summary>
/// Taken from https://github.com/icsharpcode/SharpZipLib/wiki/GZip-and-Tar-Samples.
/// Taken from
/// https://github.com/icsharpcode/SharpZipLib/wiki/GZip-and-Tar-Samples.
/// </summary>
private static void Unzip(string path, string targetDir)
{
Expand Down
110 changes: 69 additions & 41 deletions docs/samples/Microsoft.ML.Samples/Dynamic/TensorFlow/TextClassification.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,27 +13,35 @@ public static class TextClassification
/// </summary>
public static void Example()
{
string modelLocation = Microsoft.ML.SamplesUtils.DatasetUtils.DownloadTensorFlowSentimentModel();
string modelLocation = Microsoft.ML.SamplesUtils.DatasetUtils
.DownloadTensorFlowSentimentModel();

var mlContext = new MLContext();
var data = new[] { new IMDBSentiment() {
Sentiment_Text = "this film was just brilliant casting location scenery story direction " +
"everyone's really suited the part they played and you could just imagine being there robert " +
"is an amazing actor and now the same being director father came from the same scottish " +
"island as myself so i loved the fact there was a real connection with this film the witty " +
"remarks throughout the film were great it was just brilliant so much that i bought the " +
"film as soon as it was released for and would recommend it to everyone to watch and the " +
"fly fishing was amazing really cried at the end it was so sad and you know what they say " +
"if you cry at a film it must have been good and this definitely was also to the two " +
"little boy's that played the of norman and paul they were just brilliant children are " +
"often left out of the list i think because the stars that play them all grown up are " +
"such a big profile for the whole film but these children are amazing and should be praised " +
"for what they have done don't you think the whole story was so lovely because it was true " +
"and was someone's life after all that was shared with us all" } };
Sentiment_Text = "this film was just brilliant casting location " +
"scenery story direction everyone's really suited the part they " +
"played and you could just imagine being there robert is an " +
"amazing actor and now the same being director father came from " +
"the same scottish island as myself so i loved the fact there was " +
"a real connection with this film the witty remarks throughout " +
"the film were great it was just brilliant so much that i bought " +
"the film as soon as it was released for and would recommend it " +
"to everyone to watch and the fly fishing was amazing really " +
"cried at the end it was so sad and you know what they say if you " +
"cry at a film it must have been good and this definitely was " +
"also to the two little boy's that played the of norman and paul " +
"they were just brilliant children are often left out of the " +
"list i think because the stars that play them all grown up are " +
"such a big profile for the whole film but these children are " +
"amazing and should be praised for what they have done don't you " +
"think the whole story was so lovely because it was true and was " +
"someone's life after all that was shared with us all" } };
var dataView = mlContext.Data.LoadFromEnumerable(data);

// This is the dictionary to convert words into the integer indexes.
var lookupMap = mlContext.Data.LoadFromTextFile(Path.Combine(modelLocation, "imdb_word_index.csv"),
var lookupMap = mlContext.Data.LoadFromTextFile(Path.Combine(
modelLocation, "imdb_word_index.csv"),

columns: new[]
{
new TextLoader.Column("Words", DataKind.String, 0),
Expand All @@ -43,49 +51,68 @@ public static void Example()
);

// Load the TensorFlow model once.
// - Use it for quering the schema for input and output in the model
// - Use it for quering the schema for input and output in the
// model
// - Use it for prediction in the pipeline.
var tensorFlowModel = mlContext.Model.LoadTensorFlowModel(modelLocation);
var tensorFlowModel = mlContext.Model.LoadTensorFlowModel(
modelLocation);
var schema = tensorFlowModel.GetModelSchema();
var featuresType = (VectorDataViewType)schema["Features"].Type;
Console.WriteLine("Name: {0}, Type: {1}, Shape: (-1, {2})", "Features", featuresType.ItemType.RawType, featuresType.Dimensions[0]);
var predictionType = (VectorDataViewType)schema["Prediction/Softmax"].Type;
Console.WriteLine("Name: {0}, Type: {1}, Shape: (-1, {2})", "Prediction/Softmax", predictionType.ItemType.RawType, predictionType.Dimensions[0]);

// The model expects the input feature vector to be a fixed length vector.
// In this sample, CustomMappingEstimator is used to resize variable length vector to fixed length vector.
Console.WriteLine("Name: {0}, Type: {1}, Shape: (-1, {2})", "Features",
featuresType.ItemType.RawType, featuresType.Dimensions[0]);

var predictionType = (VectorDataViewType)schema["Prediction/Softmax"]
.Type;
Comment thread
sayanshaw24 marked this conversation as resolved.
Console.WriteLine("Name: {0}, Type: {1}, Shape: (-1, {2})",
"Prediction/Softmax", predictionType.ItemType.RawType,
predictionType.Dimensions[0]);

// The model expects the input feature vector to be a fixed length
// vector.
// In this sample, CustomMappingEstimator is used to resize variable
// length vector to fixed length vector.
// The following ML.NET pipeline
// 1. tokenzies the string into words,
// 2. maps each word to an integer which is an index in the dictionary ('lookupMap'),
// 3. Resizes the integer vector to a fixed length vector using CustomMappingEstimator ('ResizeFeaturesAction')
// 2. maps each word to an integer which is an index in the
// dictionary ('lookupMap'),
// 3. Resizes the integer vector to a fixed length vector using
// CustomMappingEstimator ('ResizeFeaturesAction')
// 4. Passes the data to TensorFlow for scoring.
// 5. Retreives the 'Prediction' from TensorFlow and put it into ML.NET Pipeline
// 5. Retreives the 'Prediction' from TensorFlow and put it into
// ML.NET Pipeline

Action<IMDBSentiment, IntermediateFeatures> ResizeFeaturesAction = (i, j) =>
Action<IMDBSentiment, IntermediateFeatures> ResizeFeaturesAction = (i,

@natke natke Jun 28, 2019

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This would be clearer if the (i, j) was on the same line

j) =>
{
j.Sentiment_Text = i.Sentiment_Text;
var features = i.VariableLengthFeatures;
Array.Resize(ref features, MaxSentenceLength);
j.Features = features;
};

var model = mlContext.Transforms.Text.TokenizeIntoWords("TokenizedWords", "Sentiment_Text")
.Append(mlContext.Transforms.Conversion.MapValue("VariableLengthFeatures", lookupMap,
lookupMap.Schema["Words"], lookupMap.Schema["Ids"], "TokenizedWords"))
.Append(mlContext.Transforms.CustomMapping(ResizeFeaturesAction, "Resize"))
.Append(tensorFlowModel.ScoreTensorFlowModel("Prediction/Softmax", "Features"))
.Append(mlContext.Transforms.CopyColumns("Prediction", "Prediction/Softmax"))
.Fit(dataView);
var engine = mlContext.Model.CreatePredictionEngine<IMDBSentiment, OutputScores>(model);
var model = mlContext.Transforms.Text.TokenizeIntoWords(

@natke natke Jun 28, 2019

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be better to try and align the Append methods, as well as keep the transform strings together. Something like:

    var model =
    mlContext.Transforms.Text.TokenizeIntoWords(
        "TokenizedWords",
        "Sentiment_Text")
    .Append(mlContext.Transforms.Conversion.MapValue(
        "VariableLengthFeatures",
        lookupMap,
        lookupMap.Schema["Words"],
        lookupMap.Schema["Ids"],
        "TokenizedWords"))
    .Append(mlContext.Transforms.CustomMapping(
        ResizeFeaturesAction,
        "Resize"))
    .Append(tensorFlowModel.ScoreTensorFlowModel(
        "Prediction/Softmax",
        "Features"))
    .Append(mlContext.Transforms.CopyColumns(
        "Prediction",
        "Prediction/Softmax"))
    .Fit(dataView);

"TokenizedWords", "Sentiment_Text")
.Append(mlContext.Transforms.Conversion.MapValue(
"VariableLengthFeatures", lookupMap, lookupMap.Schema["Words"],
lookupMap.Schema["Ids"], "TokenizedWords")).Append(mlContext.
Transforms.CustomMapping(ResizeFeaturesAction,"Resize"))
.Append(tensorFlowModel.ScoreTensorFlowModel("Prediction/Softmax",
"Features")).Append(mlContext.Transforms
.CopyColumns("Prediction", "Prediction/Softmax")).Fit(dataView);
var engine = mlContext.Model.CreatePredictionEngine<IMDBSentiment,
OutputScores>(model);

// Predict with TensorFlow pipeline.
var prediction = engine.Predict(data[0]);

Console.WriteLine("Number of classes: {0}", prediction.Prediction.Length);
Console.WriteLine("Is sentiment/review positive? {0}", prediction.Prediction[1] > 0.5 ? "Yes." : "No.");
Console.WriteLine("Prediction Confidence: {0}", prediction.Prediction[1].ToString("0.00"));
Console.WriteLine("Number of classes: {0}", prediction.Prediction
.Length);
Console.WriteLine("Is sentiment/review positive? {0}", prediction
.Prediction[1] > 0.5 ? "Yes." : "No.");
Console.WriteLine("Prediction Confidence: {0}", prediction.Prediction[1]
.ToString("0.00"));

/////////////////////////////////// Expected output ///////////////////////////////////
///////////////////////////// Expected output //////////////////////////
//
// Name: Features, Type: System.Int32, Shape: (-1, 600)
// Name: Prediction/Softmax, Type: System.Single, Shape: (-1, 2)
Expand All @@ -105,8 +132,9 @@ public class IMDBSentiment

/// <summary>
/// This is a variable length vector designated by VectorType attribute.
/// Variable length vectors are produced by applying operations such as 'TokenizeWords' on strings
/// resulting in vectors of tokens of variable lengths.
/// Variable length vectors are produced by applying operations such as
/// 'TokenizeWords' on strings resulting in vectors of tokens of
/// variable lengths.
/// </summary>
[VectorType]
public int[] VariableLengthFeatures { get; set; }
Expand Down
33 changes: 21 additions & 12 deletions docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/AnomalyDetection/RandomizedPcaSample.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@ public static class RandomizedPcaSample
{
public static void Example()
{
// Create a new context for ML.NET operations. It can be used for exception tracking and logging,
// as a catalog of available operations and as the source of randomness.
// Setting the seed to a fixed number in this example to make outputs deterministic.
// Create a new context for ML.NET operations. It can be used for except
// ion tracking and logging, as a catalog of available operations and as
// the source of randomness. Setting the seed to a fixed number in this
// example to make outputs deterministic.
var mlContext = new MLContext(seed: 0);

// Training data.
Expand All @@ -26,11 +27,15 @@ public static void Example()
new DataPoint(){ Features = new float[3] {-100, 50, -100} }
};

// Convert the List<DataPoint> to IDataView, a consumble format to ML.NET functions.
// Convert the List<DataPoint> to IDataView, a consumble format to
// ML.NET functions.
var data = mlContext.Data.LoadFromEnumerable(samples);

// Create an anomaly detector. Its underlying algorithm is randomized PCA.
var pipeline = mlContext.AnomalyDetection.Trainers.RandomizedPca(featureColumnName: nameof(DataPoint.Features), rank: 1, ensureZeroMean: false);
// Create an anomaly detector. Its underlying algorithm is randomized
// PCA.
var pipeline = mlContext.AnomalyDetection.Trainers.RandomizedPca(
featureColumnName: nameof(DataPoint.Features), rank: 1,
ensureZeroMean: false);

// Train the anomaly detector.
var model = pipeline.Fit(data);
Expand All @@ -39,7 +44,8 @@ public static void Example()
var transformed = model.Transform(data);

// Read ML.NET predictions into IEnumerable<Result>.
var results = mlContext.Data.CreateEnumerable<Result>(transformed, reuseRowObject: false).ToList();
var results = mlContext.Data.CreateEnumerable<Result>(transformed,
reuseRowObject: false).ToList();

// Let's go through all predictions.
for (int i = 0; i < samples.Count; ++i)
Expand All @@ -52,12 +58,14 @@ public static void Example()

if (result.PredictedLabel)
// The i-th sample is predicted as an inlier.
Console.WriteLine("The {0}-th example with features [{1}] is an inlier with a score of being inlier {2}",
i, featuresInText, result.Score);
Console.WriteLine("The {0}-th example with features [{1}]" +
"is an inlier with a score of being inlier {2}", i,
featuresInText, result.Score);
Comment thread
sayanshaw24 marked this conversation as resolved.
else
// The i-th sample is predicted as an outlier.
Console.WriteLine("The {0}-th example with features [{1}] is an outlier with a score of being inlier {2}",
i, featuresInText, result.Score);
Console.WriteLine("The {0}-th example with features [{1}] is" +
"an outlier with a score of being inlier {2}", i,
featuresInText, result.Score);
}
// Lines printed out should be
// The 0 - th example with features[1, 0, 0] is an inlier with a score of being inlier 0.7453707
Expand All @@ -68,7 +76,8 @@ public static void Example()
// The 5 - th example with features[-100, 50, -100] is an outlier with a score of being inlier 0
}

// Example with 3 feature values. A training data set is a collection of such examples.
// Example with 3 feature values. A training data set is a collection of
// such examples.
private class DataPoint
{
[VectorType(3)]
Expand Down
Loading