Skip to content

word embedding transform #545

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Ivanidzo4ka merged 29 commits into from
Jul 31, 2018
Merged

word embedding transform #545

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
5508da3
first iteration
Jun 14, 2018
04eecaa
merge with master
Jul 17, 2018
5069792
after merge commit
Jul 17, 2018
4863d9e
update entrypoints, add test
Jul 17, 2018
547d221
merge with master
Jul 20, 2018
68696f4
update documentation
Jul 20, 2018
2492597
update doc.xml
Jul 20, 2018
2048722
add license attributes.
Jul 20, 2018
42899d3
.net header!
Jul 20, 2018
de7d35f
"
Jul 20, 2018
8589f56
one more test, better wording in documentation.
Jul 21, 2018
c7087c7
SSWE!
Jul 21, 2018
f169932
Merge branch 'master' into ivanidze/wordembedding
Jul 23, 2018
930322a
fix test
Jul 23, 2018
e6b638d
better wording in doc.xml
Jul 23, 2018
df2fc46
address lowercase
Jul 23, 2018
3d787d6
fix test
Jul 23, 2018
f69e4c8
fill min/max properly
Jul 23, 2018
8df3928
proper fix for min/max
Jul 23, 2018
47fd24f
Merge branch 'master' into ivanidze/wordembedding
Jul 23, 2018
17cb142
reuse dst array.
Jul 23, 2018
a77a709
float instead of int
Jul 23, 2018
e75b892
Address more comments from Justin
Jul 24, 2018
3a0f0a2
remove unneccessary code duplication
Jul 24, 2018
9b4d989
revert code changes due concerns from justin
Jul 25, 2018
04d8da3
Merge branch 'master' into ivanidze/wordembedding
Jul 26, 2018
0bc1d59
Merge branch 'ivanidze/wordembedding' of https://github.com/Ivanidzo4…
Jul 26, 2018
9c76b6d
trailing whitespace
Jul 26, 2018
ec99c14
replace call to EnsureSize with direct array allocation
Jul 27, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 40 additions & 20 deletions src/Microsoft.ML.Transforms/EntryPoints/TextAnalytics.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@ namespace Microsoft.ML.Runtime.Transforms
/// </summary>
public static class TextAnalytics
{
[TlcModule.EntryPoint(Name = "Transforms.TextFeaturizer",
Desc = Data.TextTransform.Summary,
UserName = Data.TextTransform.UserName,
[TlcModule.EntryPoint(Name = "Transforms.TextFeaturizer",
Desc = Data.TextTransform.Summary,
UserName = Data.TextTransform.UserName,
ShortName = Data.TextTransform.LoaderSignature,
XmlInclude = new[] { @"<include file='../Microsoft.ML.Transforms/Text/doc.xml' path='doc/members/member[@name=""TextTransform""]/*' />" ,
@"<include file='../Microsoft.ML.Transforms/Text/doc.xml' path='doc/members/example[@name=""TextTransform""]/*' />"})]
Expand All @@ -34,9 +34,9 @@ public static CommonOutputs.TransformOutput TextTransform(IHostEnvironment env,
};
}

[TlcModule.EntryPoint(Name = "Transforms.WordTokenizer",
[TlcModule.EntryPoint(Name = "Transforms.WordTokenizer",
Desc = Data.DelimitedTokenizeTransform.Summary,
UserName = Data.DelimitedTokenizeTransform.UserName,
UserName = Data.DelimitedTokenizeTransform.UserName,
ShortName = Data.DelimitedTokenizeTransform.LoaderSignature,
XmlInclude = new[] { @"<include file='../Microsoft.ML.Transforms/Text/doc.xml' path='doc/members/member[@name=""WordTokenizer""]/*' />",
@"<include file='../Microsoft.ML.Transforms/Text/doc.xml' path='doc/members/example[@name=""WordTokenizer""]/*' />"})]
Expand All @@ -51,9 +51,9 @@ public static CommonOutputs.TransformOutput DelimitedTokenizeTransform(IHostEnvi
};
}

[TlcModule.EntryPoint(Name = "Transforms.NGramTranslator",
Desc = NgramTransform.Summary,
UserName = NgramTransform.UserName,
[TlcModule.EntryPoint(Name = "Transforms.NGramTranslator",
Desc = NgramTransform.Summary,
UserName = NgramTransform.UserName,
ShortName = NgramTransform.LoaderSignature,
XmlInclude = new[] { @"<include file='../Microsoft.ML.Transforms/Text/doc.xml' path='doc/members/member[@name=""NgramTranslator""]/*' />" })]
public static CommonOutputs.TransformOutput NGramTransform(IHostEnvironment env, NgramTransform.Arguments input)
Expand All @@ -67,9 +67,9 @@ public static CommonOutputs.TransformOutput NGramTransform(IHostEnvironment env,
};
}

[TlcModule.EntryPoint(Name = "Transforms.Dictionarizer",
Desc = Data.TermTransform.Summary,
UserName = Data.TermTransform.UserName,
[TlcModule.EntryPoint(Name = "Transforms.Dictionarizer",
Desc = Data.TermTransform.Summary,
UserName = Data.TermTransform.UserName,
ShortName = Data.TermTransform.LoaderSignature)]
public static CommonOutputs.TransformOutput TermTransform(IHostEnvironment env, TermTransform.Arguments input)
{
Expand All @@ -82,9 +82,9 @@ public static CommonOutputs.TransformOutput TermTransform(IHostEnvironment env,
};
}

[TlcModule.EntryPoint(Name = "Transforms.SentimentAnalyzer",
Desc = "Uses a pretrained sentiment model to score input strings",
UserName = SentimentAnalyzingTransform.UserName,
[TlcModule.EntryPoint(Name = "Transforms.SentimentAnalyzer",
Desc = "Uses a pretrained sentiment model to score input strings",
UserName = SentimentAnalyzingTransform.UserName,
ShortName = SentimentAnalyzingTransform.ShortName,
XmlInclude = new[] { @"<include file='../Microsoft.ML.Transforms/Text/doc.xml' path='doc/members/member[@name=""SentimentAnalyzer""]/*' />",
@"<include file='../Microsoft.ML.Transforms/Text/doc.xml' path='doc/members/example[@name=""SentimentAnalyzer""]/*' />"})]
Expand All @@ -99,9 +99,9 @@ public static CommonOutputs.TransformOutput AnalyzeSentiment(IHostEnvironment en
};
}

[TlcModule.EntryPoint(Name = "Transforms.CharacterTokenizer",
Desc = CharTokenizeTransform.Summary,
UserName = CharTokenizeTransform.UserName,
[TlcModule.EntryPoint(Name = "Transforms.CharacterTokenizer",
Desc = CharTokenizeTransform.Summary,
UserName = CharTokenizeTransform.UserName,
ShortName = CharTokenizeTransform.LoaderSignature,
XmlInclude = new[] { @"<include file='../Microsoft.ML.Transforms/Text/doc.xml' path='doc/members/member[@name=""CharacterTokenizer""]/*' />" })]
public static CommonOutputs.TransformOutput CharTokenize(IHostEnvironment env, CharTokenizeTransform.Arguments input)
Expand All @@ -118,9 +118,9 @@ public static CommonOutputs.TransformOutput CharTokenize(IHostEnvironment env, C
};
}

[TlcModule.EntryPoint(Name = "Transforms.LightLda",
Desc = LdaTransform.Summary,
UserName = LdaTransform.UserName,
[TlcModule.EntryPoint(Name = "Transforms.LightLda",
Desc = LdaTransform.Summary,
UserName = LdaTransform.UserName,
ShortName = LdaTransform.ShortName,
XmlInclude = new[] { @"<include file='../Microsoft.ML.Transforms/Text/doc.xml' path='doc/members/member[@name=""LightLDA""]/*' />",
@"<include file='../Microsoft.ML.Transforms/Text/doc.xml' path='doc/members/example[@name=""LightLDA""]/*' />" })]
Expand All @@ -137,5 +137,25 @@ public static CommonOutputs.TransformOutput LightLda(IHostEnvironment env, LdaTr
OutputData = view
};
}

[TlcModule.EntryPoint(Name = "Transforms.WordEmbeddings",
Desc = WordEmbeddingsTransform.Summary,
UserName = WordEmbeddingsTransform.UserName,
ShortName = WordEmbeddingsTransform.ShortName,
XmlInclude = new[] { @"<include file='../Microsoft.ML.Transforms/Text/doc.xml' path='doc/members/member[@name=""WordEmbeddings""]/*' />",
@"<include file='../Microsoft.ML.Transforms/Text/doc.xml' path='doc/members/example[@name=""WordEmbeddings""]/*' />" })]
public static CommonOutputs.TransformOutput WordEmbeddings(IHostEnvironment env, WordEmbeddingsTransform.Arguments input)
{
Contracts.CheckValue(env, nameof(env));
env.CheckValue(input, nameof(input));

var h = EntryPointUtils.CheckArgsAndCreateHost(env, "WordEmbeddings", input);
var view = new WordEmbeddingsTransform(h, input, input.Data);
return new CommonOutputs.TransformOutput()
{
Model = new TransformModel(h, view, input.Data),
OutputData = view
};
}
}
}
Loading