diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
index 74f3a6f2dff06..dfc6c505c8a1d 100644
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -76,7 +76,7 @@ jobs:
SPARK_VERSION: ${{ matrix.sparkVersion }}
FLINK_PROFILE: ${{ matrix.flinkProfile }}
run:
- mvn -T 2.5C clean install -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -Dspark.version="$SPARK_VERSION" -DskipTests=true -Dmaven.javadoc.skip=true -B -V
+ mvn clean install -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -Dspark.version="$SPARK_VERSION" -Pintegration-tests -DskipTests=true -B -V
- name: Quickstart Test
env:
SCALA_PROFILE: ${{ matrix.scalaProfile }}
diff --git a/docker/README.md b/docker/README.md
index 44477c8b6abbd..0851e9b5b7858 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -92,7 +92,7 @@ docker push apachehudi/hudi-hadoop_2.8.4-trinobase_368
You can also easily push the image to the Docker Hub using Docker Desktop app: go to `Images`, search for the image by
the name, and then click on the three dots and `Push to Hub`.
-
+
Note that you need to ask for permission to upload the Hudi Docker Demo images to the repositories.
diff --git a/docker/push_to_docker_hub.png b/docker/images/push_to_docker_hub.png
similarity index 100%
rename from docker/push_to_docker_hub.png
rename to docker/images/push_to_docker_hub.png
diff --git a/hudi-examples/hudi-examples-dbt/README.md b/hudi-examples/hudi-examples-dbt/README.md
index 52a6ea3ac8fa0..8fe796d37c521 100644
--- a/hudi-examples/hudi-examples-dbt/README.md
+++ b/hudi-examples/hudi-examples-dbt/README.md
@@ -1,3 +1,19 @@
+
## Testing dbt project: `hudi_examples_dbt`
This dbt project transforms demonstrates hudi integration with dbt, it has a few models to demonstrate the different ways in which you can create hudi datasets using dbt.
@@ -115,4 +131,4 @@ For more information on dbt:
- Read the [introduction to dbt](https://docs.getdbt.com/docs/introduction).
- Read the [dbt viewpoint](https://docs.getdbt.com/docs/about/viewpoint).
- Join the [dbt community](http://community.getdbt.com/).
----
\ No newline at end of file
+---
diff --git a/hudi-spark-datasource/hudi-spark2-common/pom.xml b/hudi-spark-datasource/hudi-spark2-common/pom.xml
index 9f3364a5e7a31..1cbdf7d1d8e1a 100644
--- a/hudi-spark-datasource/hudi-spark2-common/pom.xml
+++ b/hudi-spark-datasource/hudi-spark2-common/pom.xml
@@ -1,4 +1,20 @@
+
@@ -16,4 +32,4 @@
8
-
\ No newline at end of file
+
diff --git a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SyncUtilHelpers.java b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SyncUtilHelpers.java
index f40e53b80f871..def85c5b805d9 100644
--- a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SyncUtilHelpers.java
+++ b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SyncUtilHelpers.java
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.hudi.sync.common.util;
import org.apache.hudi.common.config.TypedProperties;
diff --git a/pom.xml b/pom.xml
index 29fb187d3dbea..570259b65d2a3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -140,7 +140,7 @@
2.12.10
${scala11.version}
2.11
- 0.12
+ 0.13
3.3.1
3.0.1
3.1.0
@@ -312,7 +312,7 @@
true
false
- release
+ release,integration-tests
deploy
@@ -432,6 +432,7 @@
DISCLAIMER
**/.*
**/*.json
+ **/*.hfile
**/*.log
**/*.sqltemplate
**/compose_env
@@ -444,6 +445,9 @@
**/generated-sources/**
.github/**
**/*.sql
+
+ **/*.iml
+ .mvn/**
@@ -1200,9 +1204,6 @@
true
-
- true
-
@@ -1375,7 +1376,8 @@
integration-tests
- integration-tests
+ deployArtifacts
+ true
@@ -1386,7 +1388,7 @@
true
true
- false
+ ${skipTests}
diff --git a/rfc/rfc-34/rfc-34.md b/rfc/rfc-34/rfc-34.md
index e65611d38e0ac..bb32b07cef17e 100644
--- a/rfc/rfc-34/rfc-34.md
+++ b/rfc/rfc-34/rfc-34.md
@@ -1,3 +1,19 @@
+
# Hudi BigQuery Integration
## Abstract
diff --git a/scripts/release/create_source_release.sh b/scripts/release/create_source_release.sh
index ff54706be237c..bd37c80b1e2cd 100755
--- a/scripts/release/create_source_release.sh
+++ b/scripts/release/create_source_release.sh
@@ -72,6 +72,7 @@ rsync -a \
--exclude ".idea" --exclude "*.iml" --exclude ".DS_Store" --exclude "build-target" \
--exclude "docs/content" --exclude ".rubydeps" \
--exclude "rfc" \
+ --exclude "docker/images" \
. hudi-$RELEASE_VERSION
tar czf ${RELEASE_DIR}/hudi-${RELEASE_VERSION}.src.tgz hudi-$RELEASE_VERSION
diff --git a/scripts/release/deploy_staging_jars.sh b/scripts/release/deploy_staging_jars.sh
index 37b320accc4e2..76f018e2025e9 100755
--- a/scripts/release/deploy_staging_jars.sh
+++ b/scripts/release/deploy_staging_jars.sh
@@ -73,6 +73,6 @@ fi
for v in "${ALL_VERSION_OPTS[@]}"
do
echo "Deploying to repository.apache.org with version option ${v}"
- COMMON_OPTIONS="${v} -Prelease -DskipTests -DretryFailedDeploymentCount=10 -DdeployArtifacts=true"
+ COMMON_OPTIONS="${v} -DdeployArtifacts=true -DskipTests -DretryFailedDeploymentCount=10"
$MVN clean deploy $COMMON_OPTIONS
done
diff --git a/scripts/release/validate_staged_release.sh b/scripts/release/validate_staged_release.sh
index 681cc2018637f..0e027442570a5 100755
--- a/scripts/release/validate_staged_release.sh
+++ b/scripts/release/validate_staged_release.sh
@@ -124,7 +124,7 @@ numBinaryFiles=`find . -iname '*' | xargs -I {} file -I {} | grep -va directory
if [ "$numBinaryFiles" -gt "0" ]; then
echo -e "There were non-text files in source release. Please check below\n"
find . -iname '*' | xargs -I {} file -I {} | grep -va directory | grep -v "/src/test/" | grep -va 'application/json' | grep -va 'text/' | grep -va 'application/xml'
- exit -1
+ exit 1
fi
echo -e "\t\tNo Binary Files in Source Release? - [OK]\n"
### END: Binary Files Check
@@ -134,7 +134,7 @@ echo "Checking for DISCLAIMER"
disclaimerFile="./DISCLAIMER"
if [ -f "$disclaimerFile" ]; then
echo "DISCLAIMER file should not be present "
- exit -1
+ exit 1
fi
echo -e "\t\tDISCLAIMER file exists ? [OK]\n"
@@ -144,23 +144,23 @@ licenseFile="./LICENSE"
noticeFile="./NOTICE"
if [ ! -f "$licenseFile" ]; then
echo "License file missing"
- exit -1
+ exit 1
fi
echo -e "\t\tLicense file exists ? [OK]"
if [ ! -f "$noticeFile" ]; then
echo "Notice file missing"
- exit -1
+ exit 1
fi
echo -e "\t\tNotice file exists ? [OK]\n"
### Licensing Check
echo "Performing custom Licensing Check "
-numfilesWithNoLicense=`find . -iname '*' -type f | grep -v NOTICE | grep -v LICENSE | grep -v '.json' | grep -v '.data'| grep -v '.commit' | grep -v DISCLAIMER | grep -v KEYS | grep -v '.mailmap' | grep -v '.sqltemplate' | grep -v 'ObjectSizeCalculator.java' | grep -v 'AvroConversionHelper.scala' | grep -v "fixtures" | xargs grep -L "Licensed to the Apache Software Foundation (ASF)" | wc -l`
+numfilesWithNoLicense=`find . -iname '*' -type f | grep -v NOTICE | grep -v LICENSE | grep -v '.json' | grep -v '.hfile' | grep -v '.data' | grep -v '.commit' | grep -v DISCLAIMER | grep -v KEYS | grep -v '.mailmap' | grep -v '.sqltemplate' | grep -v 'ObjectSizeCalculator.java' | grep -v 'AvroConversionHelper.scala' | grep -v "fixtures" | xargs grep -L "Licensed to the Apache Software Foundation (ASF)" | wc -l`
if [ "$numfilesWithNoLicense" -gt "0" ]; then
echo "There were some source files that did not have Apache License"
- find . -iname '*' -type f | grep -v NOTICE | grep -v LICENSE | grep -v '.json' | grep -v '.data' | grep -v '.commit' | grep -v DISCLAIMER | grep -v '.sqltemplate' | grep -v KEYS | grep -v '.mailmap' | grep -v 'ObjectSizeCalculator.java' | grep -v 'AvroConversionHelper.scala' | grep -v "fixtures" | xargs grep -L "Licensed to the Apache Software Foundation (ASF)"
- exit -1
+ find . -iname '*' -type f | grep -v NOTICE | grep -v LICENSE | grep -v '.json' | grep -v '.hfile' | grep -v '.data' | grep -v '.commit' | grep -v DISCLAIMER | grep -v '.sqltemplate' | grep -v KEYS | grep -v '.mailmap' | grep -v 'ObjectSizeCalculator.java' | grep -v 'AvroConversionHelper.scala' | grep -v "fixtures" | xargs grep -L "Licensed to the Apache Software Foundation (ASF)"
+ exit 1
fi
echo -e "\t\tLicensing Check Passed [OK]\n"