Skip to content

[HUDI-4609] Improve usability of upgrade/downgrade commands in Hudi CLI #6377

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yihua merged 1 commit into from
Aug 15, 2022
Merged

[HUDI-4609] Improve usability of upgrade/downgrade commands in Hudi CLI #6377

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 24 additions & 6 deletions hudi-cli/src/main/java/org/apache/hudi/cli/commands/UpgradeOrDowngradeCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
import org.apache.hudi.cli.utils.InputStreamConsumer;
import org.apache.hudi.cli.utils.SparkUtil;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.HoodieTableVersion;
import org.apache.hudi.common.util.StringUtils;

import org.apache.spark.launcher.SparkLauncher;
import org.springframework.shell.core.CommandMarker;
Expand All @@ -48,15 +50,16 @@ public String upgradeHoodieTable(
HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient();

SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath);
sparkLauncher.addAppArgs(SparkCommand.UPGRADE.toString(), master, sparkMemory, metaClient.getBasePath(), toVersion);
String toVersionName = getHoodieTableVersionName(toVersion, true);
sparkLauncher.addAppArgs(SparkCommand.UPGRADE.toString(), master, sparkMemory, metaClient.getBasePath(), toVersionName);
Process process = sparkLauncher.launch();
InputStreamConsumer.captureOutput(process);
int exitCode = process.waitFor();
HoodieCLI.refreshTableMetadata();
if (exitCode != 0) {
return String.format("Failed: Could not Upgrade/Downgrade Hoodie table to \"%s\".", toVersion);
return String.format("Failed: Could not Upgrade/Downgrade Hoodie table to \"%s\".", toVersionName);
}
return String.format("Hoodie table upgraded/downgraded to %s", toVersion);
return String.format("Hoodie table upgraded/downgraded to %s", toVersionName);
}

@CliCommand(value = "downgrade table", help = "Downgrades a table")
Expand All @@ -70,14 +73,29 @@ public String downgradeHoodieTable(

HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient();
SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath);
sparkLauncher.addAppArgs(SparkCommand.DOWNGRADE.toString(), master, sparkMemory, metaClient.getBasePath(), toVersion);
String toVersionName = getHoodieTableVersionName(toVersion, false);
sparkLauncher.addAppArgs(SparkCommand.DOWNGRADE.toString(), master, sparkMemory, metaClient.getBasePath(), toVersionName);
Process process = sparkLauncher.launch();
InputStreamConsumer.captureOutput(process);
int exitCode = process.waitFor();
HoodieCLI.refreshTableMetadata();
if (exitCode != 0) {
return String.format("Failed: Could not Upgrade/Downgrade Hoodie table to \"%s\".", toVersion);
return String.format("Failed: Could not Upgrade/Downgrade Hoodie table to \"%s\".", toVersionName);
}
return String.format("Hoodie table upgraded/downgraded to %s", toVersionName);
}

static String getHoodieTableVersionName(String versionOption, boolean overrideWithDefault) {
if (StringUtils.isNullOrEmpty(versionOption) && overrideWithDefault) {
return HoodieTableVersion.current().name();
}

try {
int versionCode = Integer.parseInt(versionOption);
return HoodieTableVersion.versionFromCode(versionCode).name();
} catch (NumberFormatException e) {
// The version option from the CLI is not a number, returns the original String
return versionOption;
}
return String.format("Hoodie table upgraded/downgraded to %s", toVersion);
}
}
14 changes: 14 additions & 0 deletions hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.junit.jupiter.params.provider.ValueSource;

import java.io.IOException;
import java.util.Arrays;
Expand Down Expand Up @@ -140,6 +141,19 @@ public void testUpgradeDowngradeCommand(HoodieTableVersion fromVersion, HoodieTa
}
}

@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testGetHoodieTableVersionName(boolean overrideWithDefault) {
assertEquals(overrideWithDefault ? HoodieTableVersion.current().name() : null,
UpgradeOrDowngradeCommand.getHoodieTableVersionName(null, overrideWithDefault));
assertEquals(overrideWithDefault ? HoodieTableVersion.current().name() : "",
UpgradeOrDowngradeCommand.getHoodieTableVersionName("", overrideWithDefault));
assertEquals("FIVE",
UpgradeOrDowngradeCommand.getHoodieTableVersionName("FIVE", overrideWithDefault));
assertEquals("FIVE",
UpgradeOrDowngradeCommand.getHoodieTableVersionName("5", overrideWithDefault));
}

private void verifyTableVersion(HoodieTableVersion expectedVersion) throws IOException {
metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
assertEquals(expectedVersion.versionCode(), metaClient.getTableConfig().getTableVersion().versionCode());
Expand Down