HDDS-5144. Create github check to alert when dependency tree is changed

.github/workflows/post-commit.yml

@@ -115,6 +115,33 @@ jobs:
           rm -rf ~/.m2/repository/org/apache/hadoop/hadoop-hdds*
           rm -rf ~/.m2/repository/org/apache/hadoop/hadoop-ozone*
         if: always()
+  dependency:
+    needs: compile
+    runs-on: ubuntu-18.04
+    timeout-minutes: 5
+    if: github.event_name != 'pull_request' || github.event.pull_request.draft == false
+    steps:
+      - name: Checkout project
+        uses: actions/checkout@v2
+      - name: Download compiled Ozone binaries
+        uses: actions/download-artifact@v2
+        with:
+          name: ozone-bin
+      - name: Untar binaries
+        run: |
+          mkdir dist
+          tar  -C dist --strip-components 1 -xzf hadoop-ozone*.tar.gz
+      - name: Execute tests
+        run: |
+          export OZONE_DIST_DIR=`pwd`/dist
+          ./hadoop-ozone/dev-support/checks/dependency.sh
+      - name: Archive build results
+        uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: dependency
+          path: target/dependency
+        continue-on-error: true
   acceptance:
     needs: compile
     runs-on: ubuntu-18.04

.gitignore

@@ -67,3 +67,6 @@ hadoop-ozone/recon/node_modules
 .mvn
 
 .dev-tools
+
+
+hadoop-ozone/dist/src/main/license/current.txt

hadoop-ozone/dev-support/checks/dependency.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../../.." || exit 1
+
+REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/dependency"}
+mkdir -p "$REPORT_DIR"
+REPORT_FILE="$REPORT_DIR/summary.txt"
+
+hadoop-ozone/dist/src/main/license/update-jar-report.sh current.txt
+
+cp hadoop-ozone/dist/src/main/license/jar-report.txt "$REPORT_DIR"
+cp hadoop-ozone/dist/src/main/license/current.txt "$REPORT_DIR"
+
+#implementation of sort cli is not exactly the same everywhere. It's better to sort with the same command locally
+(diff -uw <(sort hadoop-ozone/dist/src/main/license/jar-report.txt) <(sort hadoop-ozone/dist/src/main/license/current.txt) || true ) > "$REPORT_FILE"
+
+
+if [ -s "$REPORT_FILE" ]; then
+  echo ""
+   echo "Jar files are added/removed to/from the binary package."
+   echo ""
+   echo "Please update the hadoop-ozone/dist/src/main/license/bin/LICENSE.txt file with the modification"
+   echo "   AND execute hadoop-ozone/dist/src/main/license/update-jar-report.sh when you are ready (after a full build)"
+   echo ""
+   echo "Generated hadoop-ozone/dist/src/main/license/jar-report.txt file should be added to your pull-request. It will be used as the base of future comparison."
+   echo ""
+   echo "This check may also report positive for PRs if the source is not up-to-date with the base branch (eg. \`master\`).  In this case please merge the base branch into your source branch."
+   echo ""
+   echo "Changed jars:"
+   echo ""
+   cat $REPORT_FILE
+   exit 1
+fi

hadoop-ozone/dist/src/main/license/jar-report.txt

@@ -0,0 +1,235 @@
+share/ozone/lib/accessors-smart.jar
+share/ozone/lib/activation.jar
+share/ozone/lib/animal-sniffer-annotations.jar
+share/ozone/lib/annotations.jar
+share/ozone/lib/annotations.jar
+share/ozone/lib/aopalliance.jar
+share/ozone/lib/aopalliance-repackaged.jar
+share/ozone/lib/asm.jar
+share/ozone/lib/aws-java-sdk-core.jar
+share/ozone/lib/aws-java-sdk-kms.jar
+share/ozone/lib/aws-java-sdk-s3.jar
+share/ozone/lib/bcpkix-jdk15on.jar
+share/ozone/lib/bcprov-jdk15on.jar
+share/ozone/lib/bonecp.RELEASE.jar
+share/ozone/lib/cdi-api.jar
+share/ozone/lib/checker-qual.jar
+share/ozone/lib/commons-beanutils.jar
+share/ozone/lib/commons-cli.jar
+share/ozone/lib/commons-codec.jar
+share/ozone/lib/commons-collections.jar
+share/ozone/lib/commons-compress.jar
+share/ozone/lib/commons-configuration2.jar
+share/ozone/lib/commons-daemon.jar
+share/ozone/lib/commons-digester.jar
+share/ozone/lib/commons-io.jar
+share/ozone/lib/commons-lang3.jar
+share/ozone/lib/commons-logging.jar
+share/ozone/lib/commons-math3.jar
+share/ozone/lib/commons-net.jar
+share/ozone/lib/commons-pool2.jar
+share/ozone/lib/commons-text.jar
+share/ozone/lib/commons-validator.jar
+share/ozone/lib/derby.jar
+share/ozone/lib/disruptor.jar
+share/ozone/lib/dnsjava.jar
+share/ozone/lib/error_prone_annotations.jar
+share/ozone/lib/failureaccess.jar
+share/ozone/lib/FastInfoset.jar
+share/ozone/lib/grpc-api.jar
+share/ozone/lib/grpc-context.jar
+share/ozone/lib/grpc-core.jar
+share/ozone/lib/grpc-netty.jar
+share/ozone/lib/grpc-protobuf.jar
+share/ozone/lib/grpc-protobuf-lite.jar
+share/ozone/lib/grpc-stub.jar
+share/ozone/lib/gson.jar
+share/ozone/lib/guava-jre.jar
+share/ozone/lib/guice-assistedinject.jar
+share/ozone/lib/guice-bridge.jar
+share/ozone/lib/guice.jar
+share/ozone/lib/guice-multibindings.jar
+share/ozone/lib/guice-servlet.jar
+share/ozone/lib/hadoop-annotations.jar
+share/ozone/lib/hadoop-auth.jar
+share/ozone/lib/hadoop-common.jar
+share/ozone/lib/hadoop-hdds-client.jar
+share/ozone/lib/hadoop-hdds-common.jar
+share/ozone/lib/hadoop-hdds-config.jar
+share/ozone/lib/hadoop-hdds-container-service.jar
+share/ozone/lib/hadoop-hdds-docs.jar
+share/ozone/lib/hadoop-hdds-hadoop-dependency-client.jar
+share/ozone/lib/hadoop-hdds-hadoop-dependency-server.jar
+share/ozone/lib/hadoop-hdds-interface-admin.jar
+share/ozone/lib/hadoop-hdds-interface-client.jar
+share/ozone/lib/hadoop-hdds-interface-server.jar
+share/ozone/lib/hadoop-hdds-server-framework.jar
+share/ozone/lib/hadoop-hdds-server-scm.jar
+share/ozone/lib/hadoop-hdds-tools.jar
+share/ozone/lib/hadoop-hdfs-client.jar
+share/ozone/lib/hadoop-hdfs.jar
+share/ozone/lib/hadoop-ozone-client.jar
+share/ozone/lib/hadoop-ozone-common.jar
+share/ozone/lib/hadoop-ozone-csi.jar
+share/ozone/lib/hadoop-ozone-datanode.jar
+share/ozone/lib/hadoop-ozone-filesystem-common.jar
+share/ozone/lib/hadoop-ozone-filesystem-hadoop2.jar
+share/ozone/lib/hadoop-ozone-filesystem-hadoop3.jar
+share/ozone/lib/hadoop-ozone-filesystem.jar
+share/ozone/lib/hadoop-ozone-insight.jar
+share/ozone/lib/hadoop-ozone-interface-client.jar
+share/ozone/lib/hadoop-ozone-interface-storage.jar
+share/ozone/lib/hadoop-ozone-ozone-manager.jar
+share/ozone/lib/hadoop-ozone-reconcodegen.jar
+share/ozone/lib/hadoop-ozone-recon.jar
+share/ozone/lib/hadoop-ozone-s3gateway.jar
+share/ozone/lib/hadoop-ozone-tools.jar
+share/ozone/lib/hk2-api.jar
+share/ozone/lib/hk2-locator.jar
+share/ozone/lib/hk2-utils.jar
+share/ozone/lib/htrace-core4-incubating.jar
+share/ozone/lib/httpclient.jar
+share/ozone/lib/httpcore.jar
+share/ozone/lib/ion-java.jar
+share/ozone/lib/istack-commons-runtime.jar
+share/ozone/lib/j2objc-annotations.jar
+share/ozone/lib/jackson-annotations.jar
+share/ozone/lib/jackson-core.jar
+share/ozone/lib/jackson-databind.jar
+share/ozone/lib/jackson-dataformat-cbor.jar
+share/ozone/lib/jackson-dataformat-xml.jar
+share/ozone/lib/jackson-datatype-jsr310.jar
+share/ozone/lib/jackson-module-jaxb-annotations.jar
+share/ozone/lib/jaeger-client.jar
+share/ozone/lib/jaeger-core.jar
+share/ozone/lib/jaeger-thrift.jar
+share/ozone/lib/jaeger-tracerresolver.jar
+share/ozone/lib/jakarta.activation-api.jar
+share/ozone/lib/jakarta.annotation-api.jar
+share/ozone/lib/jakarta.inject.jar
+share/ozone/lib/jakarta.validation-api.jar
+share/ozone/lib/jakarta.ws.rs-api.jar
+share/ozone/lib/jakarta.xml.bind-api.jar
+share/ozone/lib/javassist-GA.jar
+share/ozone/lib/javax.activation-api.jar
+share/ozone/lib/javax.annotation-api.jar
+share/ozone/lib/javax.el-api.jar
+share/ozone/lib/javax.inject.jar
+share/ozone/lib/javax.interceptor-api.jar
+share/ozone/lib/javax.servlet-api.jar
+share/ozone/lib/jaxb-api.jar
+share/ozone/lib/jaxb-core.jar
+share/ozone/lib/jaxb-runtime.jar
+share/ozone/lib/jcip-annotations.jar
+share/ozone/lib/jersey-cdi1x.jar
+share/ozone/lib/jersey-client.jar
+share/ozone/lib/jersey-common.jar
+share/ozone/lib/jersey-container-servlet-core.jar
+share/ozone/lib/jersey-container-servlet.jar
+share/ozone/lib/jersey-core.jar
+share/ozone/lib/jersey-entity-filtering.jar
+share/ozone/lib/jersey-hk2.jar
+share/ozone/lib/jersey-media-jaxb.jar
+share/ozone/lib/jersey-media-json-jackson.jar
+share/ozone/lib/jersey-server.jar
+share/ozone/lib/jersey-server.jar
+share/ozone/lib/jersey-servlet.jar
+share/ozone/lib/jetty-http.v20201120.jar
+share/ozone/lib/jetty-io.v20201120.jar
+share/ozone/lib/jetty-security.v20201120.jar
+share/ozone/lib/jetty-server.v20201120.jar
+share/ozone/lib/jetty-servlet.v20201120.jar
+share/ozone/lib/jetty-util-ajax.v20201120.jar
+share/ozone/lib/jetty-util.v20201120.jar
+share/ozone/lib/jetty-webapp.v20201120.jar
+share/ozone/lib/jetty-xml.v20201120.jar
+share/ozone/lib/jmespath-java.jar
+share/ozone/lib/joda-time.jar
+share/ozone/lib/jooq-codegen.jar
+share/ozone/lib/jooq.jar
+share/ozone/lib/jooq-meta.jar
+share/ozone/lib/jsch.jar
+share/ozone/lib/json-smart.jar
+share/ozone/lib/jsp-api.jar
+share/ozone/lib/jsr305.jar
+share/ozone/lib/jsr311-api.jar
+share/ozone/lib/kerb-admin.jar
+share/ozone/lib/kerb-client.jar
+share/ozone/lib/kerb-common.jar
+share/ozone/lib/kerb-core.jar
+share/ozone/lib/kerb-crypto.jar
+share/ozone/lib/kerb-identity.jar
+share/ozone/lib/kerb-server.jar
+share/ozone/lib/kerb-simplekdc.jar
+share/ozone/lib/kerb-util.jar
+share/ozone/lib/kerby-asn1.jar
+share/ozone/lib/kerby-config.jar
+share/ozone/lib/kerby-pkix.jar
+share/ozone/lib/kerby-util.jar
+share/ozone/lib/kerby-xdr.jar
+share/ozone/lib/kotlin-stdlib-common.jar
+share/ozone/lib/kotlin-stdlib.jar
+share/ozone/lib/leveldbjni-all.jar
+share/ozone/lib/libthrift.jar
+share/ozone/lib/listenablefuture-empty-to-avoid-conflict-with-guava.jar
+share/ozone/lib/log4j-api.jar
+share/ozone/lib/log4j-core.jar
+share/ozone/lib/log4j.jar
+share/ozone/lib/metainf-services.jar
+share/ozone/lib/metrics-core.jar
+share/ozone/lib/netty-buffer.Final.jar
+share/ozone/lib/netty-codec.Final.jar
+share/ozone/lib/netty-codec-http2.Final.jar
+share/ozone/lib/netty-codec-http.Final.jar
+share/ozone/lib/netty-codec-socks.Final.jar
+share/ozone/lib/netty-common.Final.jar
+share/ozone/lib/netty-handler.Final.jar
+share/ozone/lib/netty-handler-proxy.Final.jar
+share/ozone/lib/netty-resolver.Final.jar
+share/ozone/lib/netty-transport.Final.jar
+share/ozone/lib/netty-transport-native-epoll.Final.jar
+share/ozone/lib/netty-transport-native-unix-common.Final.jar
+share/ozone/lib/nimbus-jose-jwt.jar
+share/ozone/lib/okhttp.jar
+share/ozone/lib/okio.jar
+share/ozone/lib/opentracing-api.jar
+share/ozone/lib/opentracing-noop.jar
+share/ozone/lib/opentracing-tracerresolver.jar
+share/ozone/lib/opentracing-util.jar
+share/ozone/lib/osgi-resource-locator.jar
+share/ozone/lib/perfmark-api.jar
+share/ozone/lib/picocli.jar
+share/ozone/lib/protobuf-java.jar
+share/ozone/lib/protobuf-java.jar
+share/ozone/lib/protobuf-java-util.jar
+share/ozone/lib/proto-google-common-protos.jar
+share/ozone/lib/ratis-client.jar
+share/ozone/lib/ratis-common.jar
+share/ozone/lib/ratis-grpc.jar
+share/ozone/lib/ratis-metrics.jar
+share/ozone/lib/ratis-netty.jar
+share/ozone/lib/ratis-proto.jar
+share/ozone/lib/ratis-server-api.jar
+share/ozone/lib/ratis-server.jar
+share/ozone/lib/ratis-thirdparty-misc.jar
+share/ozone/lib/ratis-tools.jar
+share/ozone/lib/re2j.jar
+share/ozone/lib/rocksdbjni.jar
+share/ozone/lib/simpleclient_common.jar
+share/ozone/lib/simpleclient_dropwizard.jar
+share/ozone/lib/simpleclient.jar
+share/ozone/lib/slf4j-api.jar
+share/ozone/lib/slf4j-log4j12.jar
+share/ozone/lib/snakeyaml.jar
+share/ozone/lib/spring-beans.RELEASE.jar
+share/ozone/lib/spring-core.RELEASE.jar
+share/ozone/lib/spring-jcl.RELEASE.jar
+share/ozone/lib/spring-jdbc.RELEASE.jar
+share/ozone/lib/spring-tx.RELEASE.jar
+share/ozone/lib/sqlite-jdbc.jar
+share/ozone/lib/stax2-api.jar
+share/ozone/lib/stax-ex.jar
+share/ozone/lib/token-provider.jar
+share/ozone/lib/txw2.jar
+share/ozone/lib/weld-servlet.Final.jar
+share/ozone/lib/woodstox-core.jar

hadoop-ozone/dist/src/main/license/update-jar-report.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+REPORT_NAME=${1:-jar-report.txt}
+
+cd "$SCRIPTDIR/../../.." || exit 1
+
+OZONE_VERSION=$(mvn help:evaluate -Dexpression=ozone.version -q -DforceStdout)
+DIST_DIR="$SCRIPTDIR/../../../target/ozone-$OZONE_VERSION"
+: ${OZONE_DIST_DIR:=$DIST_DIR}
+
+cd "$OZONE_DIST_DIR"
+
+#sed expression removes the version. Usually license is not changed with version bumps
+#jacoco and test dependencies are excluded
+find . -type f -name "*.jar" | cut -c3- | perl -wpl -e 's/-[0-9]+(.[0-9]+)*(-([0-9a-z]+-)?SNAPSHOT)?+//g' | grep -v jacoco | grep -v hadoop-hdds-test-utils | sort > "$SCRIPTDIR"/$REPORT_NAME