merge master

.github/workflows/bot.yml

@@ -18,16 +18,12 @@ jobs:
         include:
           - scala: "scala-2.11"
             spark: "spark2"
-            skipModules: ""
           - scala: "scala-2.11"
             spark: "spark2,spark-shade-unbundle-avro"
-            skipModules: ""
           - scala: "scala-2.12"
             spark: "spark3.1.x"
-            skipModules: "!hudi-spark-datasource/hudi-spark3"
           - scala: "scala-2.12"
             spark: "spark3.1.x,spark-shade-unbundle-avro"
-            skipModules: "!hudi-spark-datasource/hudi-spark3"
           - scala: "scala-2.12"
             spark: "spark3"
           - scala: "scala-2.12"
@@ -44,5 +40,4 @@ jobs:
         env:
           SCALA_PROFILE: ${{ matrix.scala }}
           SPARK_PROFILE: ${{ matrix.spark }}
-          SKIP_MODULES: ${{ matrix.skipModules }}
-        run: mvn install -P "$SCALA_PROFILE,$SPARK_PROFILE" -pl "$SKIP_MODULES" -DskipTests=true -Dmaven.javadoc.skip=true -B -V
+        run: mvn install -P "$SCALA_PROFILE,$SPARK_PROFILE" -DskipTests=true -Dmaven.javadoc.skip=true -B -V

azure-pipelines.yml

@@ -179,6 +179,8 @@ stages:
               jdkVersionOption: '1.8'
               mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
       - job: IT
+        displayName: IT modules
+        timeoutInMinutes: '90'
         steps:
           - task: AzureCLI@2
             displayName: Prepare for IT

docker/README.md

@@ -90,4 +90,98 @@ You can find more information on [Docker Hub Repositories Manual](https://docs.d
 
 ## Docker Demo Setup
 
-Please refer to the [Docker Demo Docs page](https://hudi.apache.org/docs/docker_demo).
+Please refer to the [Docker Demo Docs page](https://hudi.apache.org/docs/docker_demo).
+
+## Building Multi-Arch Images
+
+NOTE: The steps below require some code changes. Support for multi-arch builds in a fully automated manner is being
+tracked by [HUDI-3601](https://issues.apache.org/jira/browse/HUDI-3601).
+
+By default, the docker images are built for x86_64 (amd64) architecture. Docker `buildx` allows you to build multi-arch
+images, link them together with a manifest file, and push them all to a registry – with a single command. Let's say we
+want to build for arm64 architecture. First we need to ensure that `buildx` setup is done locally. Please follow the
+below steps (referred from https://www.docker.com/blog/multi-arch-images):
+
+```
+# List builders 
+~ ❯❯❯ docker buildx ls
+NAME/NODE DRIVER/ENDPOINT STATUS  PLATFORMS
+default * docker
+  default default         running linux/amd64, linux/arm64, linux/arm/v7, linux/arm/v6
+
+# If you are using the default builder, which is basically the old builder, then do following
+~ ❯❯❯ docker buildx create --name mybuilder
+mybuilder
+~ ❯❯❯ docker buildx use mybuilder
+~ ❯❯❯ docker buildx inspect --bootstrap
+[+] Building 2.5s (1/1) FINISHED
+ => [internal] booting buildkit                                                   2.5s
+ => => pulling image moby/buildkit:master                                         1.3s
+ => => creating container buildx_buildkit_mybuilder0                              1.2s
+Name:   mybuilder
+Driver: docker-container
+
+Nodes:
+Name:      mybuilder0
+Endpoint:  unix:///var/run/docker.sock
+Status:    running
+
+Platforms: linux/amd64, linux/arm64, linux/arm/v7, linux/arm/v6
+```
+
+Now goto `<HUDI_REPO_DIR>/docker/hoodie/hadoop` and change the `Dockerfile` to pull dependent images corresponding to
+arm64. For example, in [base/Dockerfile](./hoodie/hadoop/base/Dockerfile) (which pulls jdk8 image), change the
+line `FROM openjdk:8u212-jdk-slim-stretch` to `FROM arm64v8/openjdk:8u212-jdk-slim-stretch`.
+
+Then, from under `<HUDI_REPO_DIR>/docker/hoodie/hadoop` directory, execute the following command to build as well as
+push the image to the dockerhub repo:
+
+```
+# Run under hoodie/hadoop, the <tag> is optional, "latest" by default
+docker buildx build <image_folder_name> --platform <comma-separated,platforms> -t <hub-user>/<repo-name>[:<tag>] --push
+
+# For example, to build base image
+docker buildx build base --platform linux/arm64 -t apachehudi/hudi-hadoop_2.8.4-base:linux-arm64-0.10.1 --push
+```
+
+Once the base image is pushed then you could do something similar for other images.
+Change [hive](./hoodie/hadoop/hive_base/Dockerfile) dockerfile to pull the base image with tag corresponding to
+linux/arm64 platform.
+
+```
+# Change below line in the Dockerfile
+FROM apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:latest
+# as shown below
+FROM --platform=linux/arm64 apachehudi/hudi-hadoop_${HADOOP_VERSION}-base:linux-arm64-0.10.1
+
+# and then build & push from under hoodie/hadoop dir
+docker buildx build hive_base --platform linux/arm64 -t apachehudi/hudi-hadoop_2.8.4-hive_2.3.3:linux-arm64-0.10.1 --push
+```
+
+Similarly, for images that are dependent on hive (e.g. [base spark](./hoodie/hadoop/spark_base/Dockerfile)
+, [sparkmaster](./hoodie/hadoop/sparkmaster/Dockerfile), [sparkworker](./hoodie/hadoop/sparkworker/Dockerfile)
+and [sparkadhoc](./hoodie/hadoop/sparkadhoc/Dockerfile)), change the corresponding Dockerfile to pull the base hive
+image with tag corresponding to arm64. Then build and push using `docker buildx` command.
+
+For the sake of completeness, here is a [patch](https://gist.github.com/xushiyan/cec16585e884cf0693250631a1d10ec2) which
+shows what changes to make in Dockerfiles (assuming tag is named `linux-arm64-0.10.1`), and below is the list
+of `docker buildx` commands.
+
+```
+docker buildx build base --platform linux/arm64 -t apachehudi/hudi-hadoop_2.8.4-base:linux-arm64-0.10.1 --push
+docker buildx build datanode --platform linux/arm64 -t apachehudi/hudi-hadoop_2.8.4-datanode:linux-arm64-0.10.1 --push
+docker buildx build historyserver --platform linux/arm64 -t apachehudi/hudi-hadoop_2.8.4-history:linux-arm64-0.10.1 --push
+docker buildx build hive_base --platform linux/arm64 -t apachehudi/hudi-hadoop_2.8.4-hive_2.3.3:linux-arm64-0.10.1 --push
+docker buildx build namenode --platform linux/arm64 -t apachehudi/hudi-hadoop_2.8.4-namenode:linux-arm64-0.10.1 --push
+docker buildx build prestobase --platform linux/arm64 -t apachehudi/hudi-hadoop_2.8.4-prestobase_0.217:linux-arm64-0.10.1 --push
+docker buildx build spark_base --platform linux/arm64 -t apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkbase_2.4.4:linux-arm64-0.10.1 --push
+docker buildx build sparkadhoc --platform linux/arm64 -t apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkadhoc_2.4.4:linux-arm64-0.10.1 --push
+docker buildx build sparkmaster --platform linux/arm64 -t apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkmaster_2.4.4:linux-arm64-0.10.1 --push
+docker buildx build sparkworker --platform linux/arm64 -t apachehudi/hudi-hadoop_2.8.4-hive_2.3.3-sparkworker_2.4.4:linux-arm64-0.10.1 --push
+```
+
+Once all the required images are pushed to the dockerhub repos, then we need to do one additional change
+in [docker compose](./compose/docker-compose_hadoop284_hive233_spark244.yml) file.
+Apply [this patch](https://gist.github.com/codope/3dd986de5e54f0650dd74b6032e4456c) to the docker compose file so
+that [setup_demo](./setup_demo.sh) pulls images with the correct tag for arm64. And now we should be ready to run the
+setup script and follow the docker demo.

docker/demo/trino-batch1.commands

@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+select symbol, max(ts) from stock_ticks_cow group by symbol HAVING symbol = 'GOOG';
+select symbol, max(ts) from stock_ticks_mor_ro group by symbol HAVING symbol = 'GOOG';
+select symbol, ts, volume, open, close  from stock_ticks_cow where  symbol = 'GOOG';
+select symbol, ts, volume, open, close  from stock_ticks_mor_ro where  symbol = 'GOOG';

docker/demo/trino-batch2-after-compaction.commands

@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+select symbol, max(ts) from stock_ticks_mor_ro group by symbol HAVING symbol = 'GOOG';
+select symbol, ts, volume, open, close  from stock_ticks_mor_ro where  symbol = 'GOOG';

docker/demo/trino-table-check.commands

@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+show tables;

hudi-cli/src/main/java/org/apache/hudi/cli/commands/MetadataCommand.java

@@ -108,7 +108,9 @@ public String set(@CliOption(key = {"metadataDir"},
   }
 
   @CliCommand(value = "metadata create", help = "Create the Metadata Table if it does not exist")
-  public String create() throws IOException {
+  public String create(
+      @CliOption(key = "sparkMaster", unspecifiedDefaultValue = SparkUtil.DEFAULT_SPARK_MASTER, help = "Spark master") final String master
+  ) throws IOException {
     HoodieCLI.getTableMetaClient();
     Path metadataPath = new Path(getMetadataTableBasePath(HoodieCLI.basePath));
     try {
@@ -123,7 +125,7 @@ public String create() throws IOException {
 
     HoodieTimer timer = new HoodieTimer().startTimer();
     HoodieWriteConfig writeConfig = getWriteConfig();
-    initJavaSparkContext();
+    initJavaSparkContext(Option.of(master));
     SparkHoodieBackedTableMetadataWriter.create(HoodieCLI.conf, writeConfig, new HoodieSparkEngineContext(jsc));
     return String.format("Created Metadata Table in %s (duration=%.2f secs)", metadataPath, timer.endTimer() / 1000.0);
   }
@@ -145,7 +147,8 @@ public String delete() throws Exception {
   }
 
   @CliCommand(value = "metadata init", help = "Update the metadata table from commits since the creation")
-  public String init(@CliOption(key = {"readonly"}, unspecifiedDefaultValue = "false",
+  public String init(@CliOption(key = "sparkMaster", unspecifiedDefaultValue = SparkUtil.DEFAULT_SPARK_MASTER, help = "Spark master") final String master,
+                     @CliOption(key = {"readonly"}, unspecifiedDefaultValue = "false",
       help = "Open in read-only mode") final boolean readOnly) throws Exception {
     HoodieCLI.getTableMetaClient();
     Path metadataPath = new Path(getMetadataTableBasePath(HoodieCLI.basePath));
@@ -159,7 +162,7 @@ public String init(@CliOption(key = {"readonly"}, unspecifiedDefaultValue = "fal
     HoodieTimer timer = new HoodieTimer().startTimer();
     if (!readOnly) {
       HoodieWriteConfig writeConfig = getWriteConfig();
-      initJavaSparkContext();
+      initJavaSparkContext(Option.of(master));
       SparkHoodieBackedTableMetadataWriter.create(HoodieCLI.conf, writeConfig, new HoodieSparkEngineContext(jsc));
     }
 
@@ -191,9 +194,11 @@ public String stats() throws IOException {
   }
 
   @CliCommand(value = "metadata list-partitions", help = "List all partitions from metadata")
-  public String listPartitions() throws IOException {
+  public String listPartitions(
+      @CliOption(key = "sparkMaster", unspecifiedDefaultValue = SparkUtil.DEFAULT_SPARK_MASTER, help = "Spark master") final String master
+  ) throws IOException {
     HoodieCLI.getTableMetaClient();
-    initJavaSparkContext();
+    initJavaSparkContext(Option.of(master));
     HoodieMetadataConfig config = HoodieMetadataConfig.newBuilder().enable(true).build();
     HoodieBackedTableMetadata metadata = new HoodieBackedTableMetadata(new HoodieSparkEngineContext(jsc), config,
         HoodieCLI.basePath, "/tmp");
@@ -357,9 +362,9 @@ private HoodieWriteConfig getWriteConfig() {
         .withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(true).build()).build();
   }
 
-  private void initJavaSparkContext() {
+  private void initJavaSparkContext(Option<String> userDefinedMaster) {
     if (jsc == null) {
-      jsc = SparkUtil.initJavaSparkConf(SparkUtil.getDefaultConf("HoodieCLI", Option.empty()));
+      jsc = SparkUtil.initJavaSparkConf(SparkUtil.getDefaultConf("HoodieCLI", userDefinedMaster));
     }
   }
 }

hudi-cli/src/main/java/org/apache/hudi/cli/commands/SavepointsCommand.java

@@ -23,20 +23,11 @@
 import org.apache.hudi.cli.HoodieTableHeaderFields;
 import org.apache.hudi.cli.utils.InputStreamConsumer;
 import org.apache.hudi.cli.utils.SparkUtil;
-import org.apache.hudi.client.SparkRDDWriteClient;
-import org.apache.hudi.client.common.HoodieSparkEngineContext;
-import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy;
 import org.apache.hudi.common.table.HoodieTableMetaClient;
 import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
 import org.apache.hudi.common.table.timeline.HoodieInstant;
 import org.apache.hudi.common.table.timeline.HoodieTimeline;
-import org.apache.hudi.config.HoodieCompactionConfig;
-import org.apache.hudi.config.HoodieIndexConfig;
-import org.apache.hudi.config.HoodieWriteConfig;
 import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.index.HoodieIndex;
-
-import org.apache.spark.api.java.JavaSparkContext;
 import org.apache.spark.launcher.SparkLauncher;
 import org.springframework.shell.core.CommandMarker;
 import org.springframework.shell.core.annotation.CliCommand;
@@ -162,13 +153,4 @@ public String deleteSavepoint(@CliOption(key = {"commit"}, help = "Delete a save
     }
     return String.format("Savepoint \"%s\" deleted.", instantTime);
   }
-
-  private static SparkRDDWriteClient createHoodieClient(JavaSparkContext jsc, String basePath) throws Exception {
-    HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(basePath)
-        .withCompactionConfig(HoodieCompactionConfig.newBuilder()
-            .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.NEVER).build())
-        .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BLOOM).build()).build();
-    return new SparkRDDWriteClient(new HoodieSparkEngineContext(jsc), config);
-  }
-
 }

hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/HoodieAsyncService.java

@@ -170,6 +170,7 @@ private void shutdownCallback(Function<Boolean, Boolean> callback) {
       if (null != callback) {
         callback.apply(null != error);
       }
+      this.started = false;
     });
   }