Skip to content

[SPARK-20899][PySpark] PySpark supports stringIndexerOrderType in RFormula #18122

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
actuaryzhang wants to merge 6 commits into from
Closed

[SPARK-20899][PySpark] PySpark supports stringIndexerOrderType in RFormula #18122

Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4bca4d9
Pyhton port for Rformula stringIndexerOrderType
actuaryzhang May 26, 2017
c3f4430
fix doc issue
actuaryzhang May 26, 2017
3510e24
move test to test file
actuaryzhang May 29, 2017
320203e
update test
actuaryzhang May 29, 2017
4af4b35
fix test issues
actuaryzhang May 30, 2017
2e854a8
improve tests
actuaryzhang May 30, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 40 additions & 5 deletions python/pyspark/ml/feature.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3032,6 +3032,18 @@ class RFormula(JavaEstimator, HasFeaturesCol, HasLabelCol, JavaMLReadable, JavaM
...
>>> str(loadedModel)
'RFormulaModel(ResolvedRFormula(label=y, terms=[x,s], hasIntercept=true)) (uid=...)'
>>> rf = RFormula(formula="y ~ x + s", stringIndexerOrderType="alphabetDesc")
>>> rf.getStringIndexerOrderType()
'alphabetDesc'
>>> rf.fit(df).transform(df).show()
+---+---+---+---------+-----+
| y| x| s| features|label|
+---+---+---+---------+-----+
|1.0|1.0| a|[1.0,0.0]| 1.0|
|0.0|2.0| b|[2.0,1.0]| 0.0|
|0.0|0.0| a|(2,[],[])| 0.0|
+---+---+---+---------+-----+
...
Copy link
Contributor

@yanboliang yanboliang May 28, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you move the newly added test to tests.py? We keep the basic doc tests here both for test and example, other tests should be placed at tests.py. Thanks.


.. versionadded:: 1.5.0
"""
Expand All @@ -3043,26 +3055,35 @@ class RFormula(JavaEstimator, HasFeaturesCol, HasLabelCol, JavaMLReadable, JavaM
"Force to index label whether it is numeric or string",
typeConverter=TypeConverters.toBoolean)

stringIndexerOrderType = Param(Params._dummy(), "stringIndexerOrderType",
"How to order categories of a string FEATURE column used by " +
Copy link
Member

@felixcheung felixcheung May 26, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FEATURE capitalize is common here?

Copy link
Contributor Author

@actuaryzhang actuaryzhang May 26, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Changed it to lower case now.

"StringIndexer. The last category after ordering is dropped " +
"when encoding strings. Supported options: frequencyDesc, " +
"frequencyAsc, alphabetDesc, alphabetAsc. The default value " +
"is frequencyDesc. When the ordering is set to alphabetDesc, " +
"RFormula drops the same category as R when encoding strings.",
typeConverter=TypeConverters.toString)

@keyword_only
def __init__(self, formula=None, featuresCol="features", labelCol="label",
forceIndexLabel=False):
forceIndexLabel=False, stringIndexerOrderType="frequencyDesc"):
"""
__init__(self, formula=None, featuresCol="features", labelCol="label", \
forceIndexLabel=False)
forceIndexLabel=False, stringIndexerOrderType="frequencyDesc")
"""
super(RFormula, self).__init__()
self._java_obj = self._new_java_obj("org.apache.spark.ml.feature.RFormula", self.uid)
self._setDefault(forceIndexLabel=False)
self._setDefault(forceIndexLabel=False, stringIndexerOrderType="frequencyDesc")
kwargs = self._input_kwargs
self.setParams(**kwargs)

@keyword_only
@since("1.5.0")
def setParams(self, formula=None, featuresCol="features", labelCol="label",
forceIndexLabel=False):
forceIndexLabel=False, stringIndexerOrderType="frequencyDesc"):
"""
setParams(self, formula=None, featuresCol="features", labelCol="label", \
forceIndexLabel=False)
forceIndexLabel=False, stringIndexerOrderType="frequencyDesc")
Sets params for RFormula.
"""
kwargs = self._input_kwargs
Expand Down Expand Up @@ -3096,6 +3117,20 @@ def getForceIndexLabel(self):
"""
return self.getOrDefault(self.forceIndexLabel)

@since("2.3.0")
def setStringIndexerOrderType(self, value):
"""
Sets the value of :py:attr:`stringIndexerOrderType`.
"""
return self._set(stringIndexerOrderType=value)

@since("2.3.0")
def getStringIndexerOrderType(self):
"""
Gets the value of :py:attr:`stringIndexerOrderType` or its default value 'frequencyDesc'.
"""
return self.getOrDefault(self.stringIndexerOrderType)

def _create_model(self, java_model):
return RFormulaModel(java_model)

Expand Down