Skip to content

[SPARK-14615][ML][FOLLOWUP] Fix Python examples to use the new ML Vector and Matrix APIs in the ML pipeline based algorithms #13393

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
HyukjinKwon wants to merge 1 commit into from
Closed

[SPARK-14615][ML][FOLLOWUP] Fix Python examples to use the new ML Vector and Matrix APIs in the ML pipeline based algorithms #13393

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion examples/src/main/python/ml/aft_survival_regression.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

# $example on$
from pyspark.ml.regression import AFTSurvivalRegression
from pyspark.mllib.linalg import Vectors
from pyspark.ml.linalg import Vectors
Copy link
Member

@jkbradley jkbradley Jun 10, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this example run for you? It seems broken (not due to your PR though). Would you mind checking to identify the last time it worked?

Traceback (most recent call last):                                              
  File "/Users/josephkb/spark/examples/src/main/python/ml/aft_survival_regression.py", line 49, in <module>
    model = aft.fit(training)
  File "/Users/josephkb/spark/python/lib/pyspark.zip/pyspark/ml/base.py", line 64, in fit
  File "/Users/josephkb/spark/python/lib/pyspark.zip/pyspark/ml/wrapper.py", line 213, in _fit
  File "/Users/josephkb/spark/python/lib/pyspark.zip/pyspark/ml/wrapper.py", line 210, in _fit_java
  File "/Users/josephkb/spark/python/lib/py4j-0.10.1-src.zip/py4j/java_gateway.py", line 933, in __call__
  File "/Users/josephkb/spark/python/lib/pyspark.zip/pyspark/sql/utils.py", line 79, in deco
pyspark.sql.utils.IllegalArgumentException: u'requirement failed: The number of instances should be greater than 0.0, but got 0.'

# $example off$
from pyspark.sql import SparkSession

Expand Down
2 changes: 1 addition & 1 deletion examples/src/main/python/ml/chisq_selector_example.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
from pyspark.sql import SparkSession
# $example on$
from pyspark.ml.feature import ChiSqSelector
from pyspark.mllib.linalg import Vectors
from pyspark.ml.linalg import Vectors
# $example off$

if __name__ == "__main__":
Expand Down
2 changes: 1 addition & 1 deletion examples/src/main/python/ml/dct_example.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

# $example on$
from pyspark.ml.feature import DCT
from pyspark.mllib.linalg import Vectors
from pyspark.ml.linalg import Vectors
# $example off$
from pyspark.sql import SparkSession

Expand Down
2 changes: 1 addition & 1 deletion examples/src/main/python/ml/elementwise_product_example.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

# $example on$
from pyspark.ml.feature import ElementwiseProduct
from pyspark.mllib.linalg import Vectors
from pyspark.ml.linalg import Vectors
# $example off$
from pyspark.sql import SparkSession

Expand Down
2 changes: 1 addition & 1 deletion examples/src/main/python/ml/estimator_transformer_param_example.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
"""

# $example on$
from pyspark.mllib.linalg import Vectors
from pyspark.ml.linalg import Vectors
from pyspark.ml.classification import LogisticRegression
# $example off$
from pyspark.sql import SparkSession
Expand Down
2 changes: 1 addition & 1 deletion examples/src/main/python/ml/pca_example.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

# $example on$
from pyspark.ml.feature import PCA
from pyspark.mllib.linalg import Vectors
from pyspark.ml.linalg import Vectors
# $example off$
from pyspark.sql import SparkSession

Expand Down
2 changes: 1 addition & 1 deletion examples/src/main/python/ml/polynomial_expansion_example.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

# $example on$
from pyspark.ml.feature import PolynomialExpansion
from pyspark.mllib.linalg import Vectors
from pyspark.ml.linalg import Vectors
# $example off$
from pyspark.sql import SparkSession

Expand Down
19 changes: 9 additions & 10 deletions examples/src/main/python/ml/simple_params_example.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,8 @@
import sys

from pyspark.ml.classification import LogisticRegression
from pyspark.mllib.linalg import DenseVector
from pyspark.mllib.regression import LabeledPoint
from pyspark.sql import SparkSession
from pyspark.ml.linalg import DenseVector
from pyspark.sql import Row, SparkSession

"""
A simple example demonstrating ways to specify parameters for Estimators and Transformers.
Expand All @@ -42,10 +41,10 @@
# A LabeledPoint is an Object with two fields named label and features
# and Spark SQL identifies these fields and creates the schema appropriately.
training = spark.createDataFrame([
LabeledPoint(1.0, DenseVector([0.0, 1.1, 0.1])),
LabeledPoint(0.0, DenseVector([2.0, 1.0, -1.0])),
LabeledPoint(0.0, DenseVector([2.0, 1.3, 1.0])),
LabeledPoint(1.0, DenseVector([0.0, 1.2, -0.5]))])
Row(label=1.0, features=DenseVector([0.0, 1.1, 0.1])),
Row(label=0.0, features=DenseVector([2.0, 1.0, -1.0])),
Row(label=0.0, features=DenseVector([2.0, 1.3, 1.0])),
Row(label=1.0, features=DenseVector([0.0, 1.2, -0.5]))])

# Create a LogisticRegression instance with maxIter = 10.
# This instance is an Estimator.
Expand Down Expand Up @@ -77,9 +76,9 @@

# prepare test data.
test = spark.createDataFrame([
LabeledPoint(1.0, DenseVector([-1.0, 1.5, 1.3])),
LabeledPoint(0.0, DenseVector([3.0, 2.0, -0.1])),
LabeledPoint(0.0, DenseVector([0.0, 2.2, -1.5]))])
Row(label=1.0, features=DenseVector([-1.0, 1.5, 1.3])),
Row(label=0.0, features=DenseVector([3.0, 2.0, -0.1])),
Row(label=0.0, features=DenseVector([0.0, 2.2, -1.5]))])

# Make predictions on test data using the Transformer.transform() method.
# LogisticRegressionModel.transform will only use the 'features' column.
Expand Down
2 changes: 1 addition & 1 deletion examples/src/main/python/ml/vector_assembler_example.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
from __future__ import print_function

# $example on$
from pyspark.mllib.linalg import Vectors
from pyspark.ml.linalg import Vectors
from pyspark.ml.feature import VectorAssembler
# $example off$
from pyspark.sql import SparkSession
Expand Down
2 changes: 1 addition & 1 deletion examples/src/main/python/ml/vector_slicer_example.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

# $example on$
from pyspark.ml.feature import VectorSlicer
from pyspark.mllib.linalg import Vectors
from pyspark.ml.linalg import Vectors
from pyspark.sql.types import Row
# $example off$
from pyspark.sql import SparkSession
Expand Down