diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml index 74f3a6f2dff06..dfc6c505c8a1d 100644 --- a/.github/workflows/bot.yml +++ b/.github/workflows/bot.yml @@ -76,7 +76,7 @@ jobs: SPARK_VERSION: ${{ matrix.sparkVersion }} FLINK_PROFILE: ${{ matrix.flinkProfile }} run: - mvn -T 2.5C clean install -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -Dspark.version="$SPARK_VERSION" -DskipTests=true -Dmaven.javadoc.skip=true -B -V + mvn clean install -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -Dspark.version="$SPARK_VERSION" -Pintegration-tests -DskipTests=true -B -V - name: Quickstart Test env: SCALA_PROFILE: ${{ matrix.scalaProfile }} diff --git a/docker/README.md b/docker/README.md index 44477c8b6abbd..0851e9b5b7858 100644 --- a/docker/README.md +++ b/docker/README.md @@ -92,7 +92,7 @@ docker push apachehudi/hudi-hadoop_2.8.4-trinobase_368 You can also easily push the image to the Docker Hub using Docker Desktop app: go to `Images`, search for the image by the name, and then click on the three dots and `Push to Hub`. -![Push to Docker Hub](push_to_docker_hub.png) +![Push to Docker Hub](images/push_to_docker_hub.png) Note that you need to ask for permission to upload the Hudi Docker Demo images to the repositories. diff --git a/docker/push_to_docker_hub.png b/docker/images/push_to_docker_hub.png similarity index 100% rename from docker/push_to_docker_hub.png rename to docker/images/push_to_docker_hub.png diff --git a/hudi-examples/hudi-examples-dbt/README.md b/hudi-examples/hudi-examples-dbt/README.md index 52a6ea3ac8fa0..8fe796d37c521 100644 --- a/hudi-examples/hudi-examples-dbt/README.md +++ b/hudi-examples/hudi-examples-dbt/README.md @@ -1,3 +1,19 @@ + ## Testing dbt project: `hudi_examples_dbt` This dbt project transforms demonstrates hudi integration with dbt, it has a few models to demonstrate the different ways in which you can create hudi datasets using dbt. @@ -115,4 +131,4 @@ For more information on dbt: - Read the [introduction to dbt](https://docs.getdbt.com/docs/introduction). - Read the [dbt viewpoint](https://docs.getdbt.com/docs/about/viewpoint). - Join the [dbt community](http://community.getdbt.com/). ---- \ No newline at end of file +--- diff --git a/hudi-spark-datasource/hudi-spark2-common/pom.xml b/hudi-spark-datasource/hudi-spark2-common/pom.xml index 9f3364a5e7a31..1cbdf7d1d8e1a 100644 --- a/hudi-spark-datasource/hudi-spark2-common/pom.xml +++ b/hudi-spark-datasource/hudi-spark2-common/pom.xml @@ -1,4 +1,20 @@ + @@ -16,4 +32,4 @@ 8 - \ No newline at end of file + diff --git a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SyncUtilHelpers.java b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SyncUtilHelpers.java index f40e53b80f871..def85c5b805d9 100644 --- a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SyncUtilHelpers.java +++ b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SyncUtilHelpers.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.hudi.sync.common.util; import org.apache.hudi.common.config.TypedProperties; diff --git a/pom.xml b/pom.xml index 29fb187d3dbea..570259b65d2a3 100644 --- a/pom.xml +++ b/pom.xml @@ -140,7 +140,7 @@ 2.12.10 ${scala11.version} 2.11 - 0.12 + 0.13 3.3.1 3.0.1 3.1.0 @@ -312,7 +312,7 @@ true false - release + release,integration-tests deploy @@ -432,6 +432,7 @@ DISCLAIMER **/.* **/*.json + **/*.hfile **/*.log **/*.sqltemplate **/compose_env @@ -444,6 +445,9 @@ **/generated-sources/** .github/** **/*.sql + + **/*.iml + .mvn/** @@ -1200,9 +1204,6 @@ true - - true - @@ -1375,7 +1376,8 @@ integration-tests - integration-tests + deployArtifacts + true @@ -1386,7 +1388,7 @@ true true - false + ${skipTests} diff --git a/rfc/rfc-34/rfc-34.md b/rfc/rfc-34/rfc-34.md index e65611d38e0ac..bb32b07cef17e 100644 --- a/rfc/rfc-34/rfc-34.md +++ b/rfc/rfc-34/rfc-34.md @@ -1,3 +1,19 @@ + # Hudi BigQuery Integration ## Abstract diff --git a/scripts/release/create_source_release.sh b/scripts/release/create_source_release.sh index ff54706be237c..bd37c80b1e2cd 100755 --- a/scripts/release/create_source_release.sh +++ b/scripts/release/create_source_release.sh @@ -72,6 +72,7 @@ rsync -a \ --exclude ".idea" --exclude "*.iml" --exclude ".DS_Store" --exclude "build-target" \ --exclude "docs/content" --exclude ".rubydeps" \ --exclude "rfc" \ + --exclude "docker/images" \ . hudi-$RELEASE_VERSION tar czf ${RELEASE_DIR}/hudi-${RELEASE_VERSION}.src.tgz hudi-$RELEASE_VERSION diff --git a/scripts/release/deploy_staging_jars.sh b/scripts/release/deploy_staging_jars.sh index 37b320accc4e2..76f018e2025e9 100755 --- a/scripts/release/deploy_staging_jars.sh +++ b/scripts/release/deploy_staging_jars.sh @@ -73,6 +73,6 @@ fi for v in "${ALL_VERSION_OPTS[@]}" do echo "Deploying to repository.apache.org with version option ${v}" - COMMON_OPTIONS="${v} -Prelease -DskipTests -DretryFailedDeploymentCount=10 -DdeployArtifacts=true" + COMMON_OPTIONS="${v} -DdeployArtifacts=true -DskipTests -DretryFailedDeploymentCount=10" $MVN clean deploy $COMMON_OPTIONS done diff --git a/scripts/release/validate_staged_release.sh b/scripts/release/validate_staged_release.sh index 681cc2018637f..0e027442570a5 100755 --- a/scripts/release/validate_staged_release.sh +++ b/scripts/release/validate_staged_release.sh @@ -124,7 +124,7 @@ numBinaryFiles=`find . -iname '*' | xargs -I {} file -I {} | grep -va directory if [ "$numBinaryFiles" -gt "0" ]; then echo -e "There were non-text files in source release. Please check below\n" find . -iname '*' | xargs -I {} file -I {} | grep -va directory | grep -v "/src/test/" | grep -va 'application/json' | grep -va 'text/' | grep -va 'application/xml' - exit -1 + exit 1 fi echo -e "\t\tNo Binary Files in Source Release? - [OK]\n" ### END: Binary Files Check @@ -134,7 +134,7 @@ echo "Checking for DISCLAIMER" disclaimerFile="./DISCLAIMER" if [ -f "$disclaimerFile" ]; then echo "DISCLAIMER file should not be present " - exit -1 + exit 1 fi echo -e "\t\tDISCLAIMER file exists ? [OK]\n" @@ -144,23 +144,23 @@ licenseFile="./LICENSE" noticeFile="./NOTICE" if [ ! -f "$licenseFile" ]; then echo "License file missing" - exit -1 + exit 1 fi echo -e "\t\tLicense file exists ? [OK]" if [ ! -f "$noticeFile" ]; then echo "Notice file missing" - exit -1 + exit 1 fi echo -e "\t\tNotice file exists ? [OK]\n" ### Licensing Check echo "Performing custom Licensing Check " -numfilesWithNoLicense=`find . -iname '*' -type f | grep -v NOTICE | grep -v LICENSE | grep -v '.json' | grep -v '.data'| grep -v '.commit' | grep -v DISCLAIMER | grep -v KEYS | grep -v '.mailmap' | grep -v '.sqltemplate' | grep -v 'ObjectSizeCalculator.java' | grep -v 'AvroConversionHelper.scala' | grep -v "fixtures" | xargs grep -L "Licensed to the Apache Software Foundation (ASF)" | wc -l` +numfilesWithNoLicense=`find . -iname '*' -type f | grep -v NOTICE | grep -v LICENSE | grep -v '.json' | grep -v '.hfile' | grep -v '.data' | grep -v '.commit' | grep -v DISCLAIMER | grep -v KEYS | grep -v '.mailmap' | grep -v '.sqltemplate' | grep -v 'ObjectSizeCalculator.java' | grep -v 'AvroConversionHelper.scala' | grep -v "fixtures" | xargs grep -L "Licensed to the Apache Software Foundation (ASF)" | wc -l` if [ "$numfilesWithNoLicense" -gt "0" ]; then echo "There were some source files that did not have Apache License" - find . -iname '*' -type f | grep -v NOTICE | grep -v LICENSE | grep -v '.json' | grep -v '.data' | grep -v '.commit' | grep -v DISCLAIMER | grep -v '.sqltemplate' | grep -v KEYS | grep -v '.mailmap' | grep -v 'ObjectSizeCalculator.java' | grep -v 'AvroConversionHelper.scala' | grep -v "fixtures" | xargs grep -L "Licensed to the Apache Software Foundation (ASF)" - exit -1 + find . -iname '*' -type f | grep -v NOTICE | grep -v LICENSE | grep -v '.json' | grep -v '.hfile' | grep -v '.data' | grep -v '.commit' | grep -v DISCLAIMER | grep -v '.sqltemplate' | grep -v KEYS | grep -v '.mailmap' | grep -v 'ObjectSizeCalculator.java' | grep -v 'AvroConversionHelper.scala' | grep -v "fixtures" | xargs grep -L "Licensed to the Apache Software Foundation (ASF)" + exit 1 fi echo -e "\t\tLicensing Check Passed [OK]\n"