diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java index 1cffdf941f917..fd09a27271a85 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java @@ -57,10 +57,18 @@ public static SparkLauncher initLauncher(String propertiesFile) throws URISyntax } File libDirectory = new File(new File(currentJar).getParent(), "lib"); - // This lib directory may be not required, such as providing libraries through a bundle jar if (libDirectory.exists()) { + // When directly using hudi-cli module, the jars under the lib directory + // generated by the compilation is required Arrays.stream(libDirectory.list()).forEach(library -> - sparkLauncher.addJar(new File(libDirectory, library).getAbsolutePath())); + sparkLauncher.addJar(new File(libDirectory, library).getAbsolutePath())); + } else { + // When using hudi-cli-bundle, we also need to add the hudi-spark*-bundle + // so that the Hudi Spark job can be launched + String sparkBundleJarPath = System.getenv("SPARK_BUNDLE_JAR"); + if (!StringUtils.isNullOrEmpty(sparkBundleJarPath)) { + sparkLauncher.addJar(sparkBundleJarPath); + } } return sparkLauncher; }