diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml index f2c8577b572a..8a02753db5dc 100644 --- a/.github/workflows/post-commit.yml +++ b/.github/workflows/post-commit.yml @@ -115,6 +115,33 @@ jobs: rm -rf ~/.m2/repository/org/apache/hadoop/hadoop-hdds* rm -rf ~/.m2/repository/org/apache/hadoop/hadoop-ozone* if: always() + dependency: + needs: compile + runs-on: ubuntu-18.04 + timeout-minutes: 5 + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false + steps: + - name: Checkout project + uses: actions/checkout@v2 + - name: Download compiled Ozone binaries + uses: actions/download-artifact@v2 + with: + name: ozone-bin + - name: Untar binaries + run: | + mkdir dist + tar -C dist --strip-components 1 -xzf hadoop-ozone*.tar.gz + - name: Execute tests + run: | + export OZONE_DIST_DIR=`pwd`/dist + ./hadoop-ozone/dev-support/checks/dependency.sh + - name: Archive build results + uses: actions/upload-artifact@v2 + if: always() + with: + name: dependency + path: target/dependency + continue-on-error: true acceptance: needs: compile runs-on: ubuntu-18.04 diff --git a/.gitignore b/.gitignore index d3a3e1b86dc9..b89e06e92ab5 100644 --- a/.gitignore +++ b/.gitignore @@ -67,3 +67,6 @@ hadoop-ozone/recon/node_modules .mvn .dev-tools + + +hadoop-ozone/dist/src/main/license/current.txt diff --git a/hadoop-ozone/dev-support/checks/dependency.sh b/hadoop-ozone/dev-support/checks/dependency.sh new file mode 100755 index 000000000000..95cddf9d6f6d --- /dev/null +++ b/hadoop-ozone/dev-support/checks/dependency.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "$DIR/../../.." || exit 1 + +REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/dependency"} +mkdir -p "$REPORT_DIR" +REPORT_FILE="$REPORT_DIR/summary.txt" + +hadoop-ozone/dist/src/main/license/update-jar-report.sh current.txt + +cp hadoop-ozone/dist/src/main/license/jar-report.txt "$REPORT_DIR" +cp hadoop-ozone/dist/src/main/license/current.txt "$REPORT_DIR" + +#implementation of sort cli is not exactly the same everywhere. It's better to sort with the same command locally +(diff -uw <(sort hadoop-ozone/dist/src/main/license/jar-report.txt) <(sort hadoop-ozone/dist/src/main/license/current.txt) || true ) > "$REPORT_FILE" + + +if [ -s "$REPORT_FILE" ]; then + echo "" + echo "Jar files are added/removed to/from the binary package." + echo "" + echo "Please update the hadoop-ozone/dist/src/main/license/bin/LICENSE.txt file with the modification" + echo " AND execute hadoop-ozone/dist/src/main/license/update-jar-report.sh when you are ready (after a full build)" + echo "" + echo "Generated hadoop-ozone/dist/src/main/license/jar-report.txt file should be added to your pull-request. It will be used as the base of future comparison." + echo "" + echo "This check may also report positive for PRs if the source is not up-to-date with the base branch (eg. \`master\`). In this case please merge the base branch into your source branch." + echo "" + echo "Changed jars:" + echo "" + cat $REPORT_FILE + exit 1 +fi diff --git a/hadoop-ozone/dist/src/main/license/jar-report.txt b/hadoop-ozone/dist/src/main/license/jar-report.txt new file mode 100644 index 000000000000..cad4e63939ac --- /dev/null +++ b/hadoop-ozone/dist/src/main/license/jar-report.txt @@ -0,0 +1,235 @@ +share/ozone/lib/accessors-smart.jar +share/ozone/lib/activation.jar +share/ozone/lib/animal-sniffer-annotations.jar +share/ozone/lib/annotations.jar +share/ozone/lib/annotations.jar +share/ozone/lib/aopalliance.jar +share/ozone/lib/aopalliance-repackaged.jar +share/ozone/lib/asm.jar +share/ozone/lib/aws-java-sdk-core.jar +share/ozone/lib/aws-java-sdk-kms.jar +share/ozone/lib/aws-java-sdk-s3.jar +share/ozone/lib/bcpkix-jdk15on.jar +share/ozone/lib/bcprov-jdk15on.jar +share/ozone/lib/bonecp.RELEASE.jar +share/ozone/lib/cdi-api.jar +share/ozone/lib/checker-qual.jar +share/ozone/lib/commons-beanutils.jar +share/ozone/lib/commons-cli.jar +share/ozone/lib/commons-codec.jar +share/ozone/lib/commons-collections.jar +share/ozone/lib/commons-compress.jar +share/ozone/lib/commons-configuration2.jar +share/ozone/lib/commons-daemon.jar +share/ozone/lib/commons-digester.jar +share/ozone/lib/commons-io.jar +share/ozone/lib/commons-lang3.jar +share/ozone/lib/commons-logging.jar +share/ozone/lib/commons-math3.jar +share/ozone/lib/commons-net.jar +share/ozone/lib/commons-pool2.jar +share/ozone/lib/commons-text.jar +share/ozone/lib/commons-validator.jar +share/ozone/lib/derby.jar +share/ozone/lib/disruptor.jar +share/ozone/lib/dnsjava.jar +share/ozone/lib/error_prone_annotations.jar +share/ozone/lib/failureaccess.jar +share/ozone/lib/FastInfoset.jar +share/ozone/lib/grpc-api.jar +share/ozone/lib/grpc-context.jar +share/ozone/lib/grpc-core.jar +share/ozone/lib/grpc-netty.jar +share/ozone/lib/grpc-protobuf.jar +share/ozone/lib/grpc-protobuf-lite.jar +share/ozone/lib/grpc-stub.jar +share/ozone/lib/gson.jar +share/ozone/lib/guava-jre.jar +share/ozone/lib/guice-assistedinject.jar +share/ozone/lib/guice-bridge.jar +share/ozone/lib/guice.jar +share/ozone/lib/guice-multibindings.jar +share/ozone/lib/guice-servlet.jar +share/ozone/lib/hadoop-annotations.jar +share/ozone/lib/hadoop-auth.jar +share/ozone/lib/hadoop-common.jar +share/ozone/lib/hadoop-hdds-client.jar +share/ozone/lib/hadoop-hdds-common.jar +share/ozone/lib/hadoop-hdds-config.jar +share/ozone/lib/hadoop-hdds-container-service.jar +share/ozone/lib/hadoop-hdds-docs.jar +share/ozone/lib/hadoop-hdds-hadoop-dependency-client.jar +share/ozone/lib/hadoop-hdds-hadoop-dependency-server.jar +share/ozone/lib/hadoop-hdds-interface-admin.jar +share/ozone/lib/hadoop-hdds-interface-client.jar +share/ozone/lib/hadoop-hdds-interface-server.jar +share/ozone/lib/hadoop-hdds-server-framework.jar +share/ozone/lib/hadoop-hdds-server-scm.jar +share/ozone/lib/hadoop-hdds-tools.jar +share/ozone/lib/hadoop-hdfs-client.jar +share/ozone/lib/hadoop-hdfs.jar +share/ozone/lib/hadoop-ozone-client.jar +share/ozone/lib/hadoop-ozone-common.jar +share/ozone/lib/hadoop-ozone-csi.jar +share/ozone/lib/hadoop-ozone-datanode.jar +share/ozone/lib/hadoop-ozone-filesystem-common.jar +share/ozone/lib/hadoop-ozone-filesystem-hadoop2.jar +share/ozone/lib/hadoop-ozone-filesystem-hadoop3.jar +share/ozone/lib/hadoop-ozone-filesystem.jar +share/ozone/lib/hadoop-ozone-insight.jar +share/ozone/lib/hadoop-ozone-interface-client.jar +share/ozone/lib/hadoop-ozone-interface-storage.jar +share/ozone/lib/hadoop-ozone-ozone-manager.jar +share/ozone/lib/hadoop-ozone-reconcodegen.jar +share/ozone/lib/hadoop-ozone-recon.jar +share/ozone/lib/hadoop-ozone-s3gateway.jar +share/ozone/lib/hadoop-ozone-tools.jar +share/ozone/lib/hk2-api.jar +share/ozone/lib/hk2-locator.jar +share/ozone/lib/hk2-utils.jar +share/ozone/lib/htrace-core4-incubating.jar +share/ozone/lib/httpclient.jar +share/ozone/lib/httpcore.jar +share/ozone/lib/ion-java.jar +share/ozone/lib/istack-commons-runtime.jar +share/ozone/lib/j2objc-annotations.jar +share/ozone/lib/jackson-annotations.jar +share/ozone/lib/jackson-core.jar +share/ozone/lib/jackson-databind.jar +share/ozone/lib/jackson-dataformat-cbor.jar +share/ozone/lib/jackson-dataformat-xml.jar +share/ozone/lib/jackson-datatype-jsr310.jar +share/ozone/lib/jackson-module-jaxb-annotations.jar +share/ozone/lib/jaeger-client.jar +share/ozone/lib/jaeger-core.jar +share/ozone/lib/jaeger-thrift.jar +share/ozone/lib/jaeger-tracerresolver.jar +share/ozone/lib/jakarta.activation-api.jar +share/ozone/lib/jakarta.annotation-api.jar +share/ozone/lib/jakarta.inject.jar +share/ozone/lib/jakarta.validation-api.jar +share/ozone/lib/jakarta.ws.rs-api.jar +share/ozone/lib/jakarta.xml.bind-api.jar +share/ozone/lib/javassist-GA.jar +share/ozone/lib/javax.activation-api.jar +share/ozone/lib/javax.annotation-api.jar +share/ozone/lib/javax.el-api.jar +share/ozone/lib/javax.inject.jar +share/ozone/lib/javax.interceptor-api.jar +share/ozone/lib/javax.servlet-api.jar +share/ozone/lib/jaxb-api.jar +share/ozone/lib/jaxb-core.jar +share/ozone/lib/jaxb-runtime.jar +share/ozone/lib/jcip-annotations.jar +share/ozone/lib/jersey-cdi1x.jar +share/ozone/lib/jersey-client.jar +share/ozone/lib/jersey-common.jar +share/ozone/lib/jersey-container-servlet-core.jar +share/ozone/lib/jersey-container-servlet.jar +share/ozone/lib/jersey-core.jar +share/ozone/lib/jersey-entity-filtering.jar +share/ozone/lib/jersey-hk2.jar +share/ozone/lib/jersey-media-jaxb.jar +share/ozone/lib/jersey-media-json-jackson.jar +share/ozone/lib/jersey-server.jar +share/ozone/lib/jersey-server.jar +share/ozone/lib/jersey-servlet.jar +share/ozone/lib/jetty-http.v20201120.jar +share/ozone/lib/jetty-io.v20201120.jar +share/ozone/lib/jetty-security.v20201120.jar +share/ozone/lib/jetty-server.v20201120.jar +share/ozone/lib/jetty-servlet.v20201120.jar +share/ozone/lib/jetty-util-ajax.v20201120.jar +share/ozone/lib/jetty-util.v20201120.jar +share/ozone/lib/jetty-webapp.v20201120.jar +share/ozone/lib/jetty-xml.v20201120.jar +share/ozone/lib/jmespath-java.jar +share/ozone/lib/joda-time.jar +share/ozone/lib/jooq-codegen.jar +share/ozone/lib/jooq.jar +share/ozone/lib/jooq-meta.jar +share/ozone/lib/jsch.jar +share/ozone/lib/json-smart.jar +share/ozone/lib/jsp-api.jar +share/ozone/lib/jsr305.jar +share/ozone/lib/jsr311-api.jar +share/ozone/lib/kerb-admin.jar +share/ozone/lib/kerb-client.jar +share/ozone/lib/kerb-common.jar +share/ozone/lib/kerb-core.jar +share/ozone/lib/kerb-crypto.jar +share/ozone/lib/kerb-identity.jar +share/ozone/lib/kerb-server.jar +share/ozone/lib/kerb-simplekdc.jar +share/ozone/lib/kerb-util.jar +share/ozone/lib/kerby-asn1.jar +share/ozone/lib/kerby-config.jar +share/ozone/lib/kerby-pkix.jar +share/ozone/lib/kerby-util.jar +share/ozone/lib/kerby-xdr.jar +share/ozone/lib/kotlin-stdlib-common.jar +share/ozone/lib/kotlin-stdlib.jar +share/ozone/lib/leveldbjni-all.jar +share/ozone/lib/libthrift.jar +share/ozone/lib/listenablefuture-empty-to-avoid-conflict-with-guava.jar +share/ozone/lib/log4j-api.jar +share/ozone/lib/log4j-core.jar +share/ozone/lib/log4j.jar +share/ozone/lib/metainf-services.jar +share/ozone/lib/metrics-core.jar +share/ozone/lib/netty-buffer.Final.jar +share/ozone/lib/netty-codec.Final.jar +share/ozone/lib/netty-codec-http2.Final.jar +share/ozone/lib/netty-codec-http.Final.jar +share/ozone/lib/netty-codec-socks.Final.jar +share/ozone/lib/netty-common.Final.jar +share/ozone/lib/netty-handler.Final.jar +share/ozone/lib/netty-handler-proxy.Final.jar +share/ozone/lib/netty-resolver.Final.jar +share/ozone/lib/netty-transport.Final.jar +share/ozone/lib/netty-transport-native-epoll.Final.jar +share/ozone/lib/netty-transport-native-unix-common.Final.jar +share/ozone/lib/nimbus-jose-jwt.jar +share/ozone/lib/okhttp.jar +share/ozone/lib/okio.jar +share/ozone/lib/opentracing-api.jar +share/ozone/lib/opentracing-noop.jar +share/ozone/lib/opentracing-tracerresolver.jar +share/ozone/lib/opentracing-util.jar +share/ozone/lib/osgi-resource-locator.jar +share/ozone/lib/perfmark-api.jar +share/ozone/lib/picocli.jar +share/ozone/lib/protobuf-java.jar +share/ozone/lib/protobuf-java.jar +share/ozone/lib/protobuf-java-util.jar +share/ozone/lib/proto-google-common-protos.jar +share/ozone/lib/ratis-client.jar +share/ozone/lib/ratis-common.jar +share/ozone/lib/ratis-grpc.jar +share/ozone/lib/ratis-metrics.jar +share/ozone/lib/ratis-netty.jar +share/ozone/lib/ratis-proto.jar +share/ozone/lib/ratis-server-api.jar +share/ozone/lib/ratis-server.jar +share/ozone/lib/ratis-thirdparty-misc.jar +share/ozone/lib/ratis-tools.jar +share/ozone/lib/re2j.jar +share/ozone/lib/rocksdbjni.jar +share/ozone/lib/simpleclient_common.jar +share/ozone/lib/simpleclient_dropwizard.jar +share/ozone/lib/simpleclient.jar +share/ozone/lib/slf4j-api.jar +share/ozone/lib/slf4j-log4j12.jar +share/ozone/lib/snakeyaml.jar +share/ozone/lib/spring-beans.RELEASE.jar +share/ozone/lib/spring-core.RELEASE.jar +share/ozone/lib/spring-jcl.RELEASE.jar +share/ozone/lib/spring-jdbc.RELEASE.jar +share/ozone/lib/spring-tx.RELEASE.jar +share/ozone/lib/sqlite-jdbc.jar +share/ozone/lib/stax2-api.jar +share/ozone/lib/stax-ex.jar +share/ozone/lib/token-provider.jar +share/ozone/lib/txw2.jar +share/ozone/lib/weld-servlet.Final.jar +share/ozone/lib/woodstox-core.jar diff --git a/hadoop-ozone/dist/src/main/license/update-jar-report.sh b/hadoop-ozone/dist/src/main/license/update-jar-report.sh new file mode 100755 index 000000000000..0d2aa5c75803 --- /dev/null +++ b/hadoop-ozone/dist/src/main/license/update-jar-report.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +REPORT_NAME=${1:-jar-report.txt} + +cd "$SCRIPTDIR/../../.." || exit 1 + +OZONE_VERSION=$(mvn help:evaluate -Dexpression=ozone.version -q -DforceStdout) +DIST_DIR="$SCRIPTDIR/../../../target/ozone-$OZONE_VERSION" +: ${OZONE_DIST_DIR:=$DIST_DIR} + +cd "$OZONE_DIST_DIR" + +#sed expression removes the version. Usually license is not changed with version bumps +#jacoco and test dependencies are excluded +find . -type f -name "*.jar" | cut -c3- | perl -wpl -e 's/-[0-9]+(.[0-9]+)*(-([0-9a-z]+-)?SNAPSHOT)?+//g' | grep -v jacoco | grep -v hadoop-hdds-test-utils | sort > "$SCRIPTDIR"/$REPORT_NAME