Skip to content

[SPARK-11744][Launcher] Fix print version throw exception when using pyspark shell #9721

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jerryshao wants to merge 2 commits into from
Closed

[SPARK-11744][Launcher] Fix print version throw exception when using pyspark shell #9721

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7f90c60
Fix print version exception on python shell
jerryshao Nov 15, 2015
0534833
Remove unnessary variable
jerryshao Nov 17, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 10 additions & 7 deletions launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder {
}

final List<String> sparkArgs;
private final boolean printHelp;
private final boolean printInfo;

/**
* Controls whether mixing spark-submit arguments with app arguments is allowed. This is needed
Expand All @@ -88,7 +88,7 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder {

SparkSubmitCommandBuilder() {
this.sparkArgs = new ArrayList<String>();
this.printHelp = false;
this.printInfo = false;
}

SparkSubmitCommandBuilder(List<String> args) {
Expand All @@ -108,14 +108,14 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder {

OptionParser parser = new OptionParser();
parser.parse(submitArgs);
this.printHelp = parser.helpRequested;
this.printInfo = parser.infoRequested;
}

@Override
public List<String> buildCommand(Map<String, String> env) throws IOException {
if (PYSPARK_SHELL_RESOURCE.equals(appResource) && !printHelp) {
if (PYSPARK_SHELL_RESOURCE.equals(appResource) && !printInfo) {
return buildPySparkShellCommand(env);
} else if (SPARKR_SHELL_RESOURCE.equals(appResource) && !printHelp) {
} else if (SPARKR_SHELL_RESOURCE.equals(appResource) && !printInfo) {
return buildSparkRCommand(env);
} else {
return buildSparkSubmitCommand(env);
Expand Down Expand Up @@ -311,7 +311,7 @@ private boolean isThriftServer(String mainClass) {

private class OptionParser extends SparkSubmitOptionParser {

boolean helpRequested = false;
boolean infoRequested = false;

@Override
protected boolean handle(String opt, String value) {
Expand Down Expand Up @@ -344,7 +344,10 @@ protected boolean handle(String opt, String value) {
appResource = specialClasses.get(value);
}
} else if (opt.equals(HELP) || opt.equals(USAGE_ERROR)) {
helpRequested = true;
infoRequested = true;
sparkArgs.add(opt);
} else if (opt.equals(VERSION)) {
infoRequested = true;
sparkArgs.add(opt);
} else {
sparkArgs.add(opt);
Expand Down