Skip to content

HDDS-11450. Avoid managing versions for transitive-only dependencies #7190

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
adoroszlai wants to merge 10 commits into from
Closed

HDDS-11450. Avoid managing versions for transitive-only dependencies #7190

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
e1d3953
HDDS-11450. Avoid managing versions for transitive-only dependencies
adoroszlai Sep 11, 2024
1f247ac
copy dependencies per project
adoroszlai Sep 11, 2024
699d60b
fix re2j version in license.exceptions
adoroszlai Sep 11, 2024
10d03cf
fix dependency check
adoroszlai Sep 11, 2024
f9c2649
exclude ozone-filesystem-shaded, which is base for hadoop2/3-specific…
adoroszlai Sep 12, 2024
91e4ad8
copy classpath files instead of extracting them; avoid using profile,…
adoroszlai Sep 12, 2024
4fd723d
restore nimbus-jose-jwt and snappy-java for CVE
adoroszlai Sep 12, 2024
098475d
Merge remote-tracking branch 'origin/master' into HDDS-11450
adoroszlai Oct 3, 2024
fd1b144
Merge remote-tracking branch 'origin/master' into HDDS-11450
adoroszlai Oct 5, 2024
a0a7bb8
do not override copy-jars from root POM
adoroszlai Oct 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 0 additions & 12 deletions hadoop-hdds/hadoop-dependency-client/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop.thirdparty</groupId>
<artifactId>hadoop-shaded-guava</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-annotations</artifactId>
Expand Down Expand Up @@ -210,10 +206,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
</exclusions>

</dependency>
<dependency>
<groupId>org.apache.hadoop.thirdparty</groupId>
<artifactId>hadoop-shaded-guava</artifactId>
</dependency>
<dependency>
<groupId>com.nimbusds</groupId>
<artifactId>nimbus-jose-jwt</artifactId>
Expand All @@ -224,10 +216,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<version>${hadoop.version}</version>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop.thirdparty</groupId>
<artifactId>hadoop-shaded-guava</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
Expand Down
12 changes: 0 additions & 12 deletions hadoop-hdds/hadoop-dependency-server/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop.thirdparty</groupId>
<artifactId>hadoop-shaded-guava</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.curator</groupId>
<artifactId>*</artifactId>
Expand Down Expand Up @@ -148,10 +144,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop.thirdparty</groupId>
<artifactId>hadoop-shaded-guava</artifactId>
</dependency>
<dependency>
<groupId>com.nimbusds</groupId>
<artifactId>nimbus-jose-jwt</artifactId>
Expand All @@ -171,10 +163,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<groupId>com.sun.jersey</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop.thirdparty</groupId>
<artifactId>hadoop-shaded-guava</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>*</artifactId>
Expand Down
31 changes: 0 additions & 31 deletions hadoop-hdds/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -322,36 +322,5 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
</plugins>
</build>
</profile>
<profile>
<id>add-classpath-descriptor</id>
<activation>
<file>
<exists>src/main/java</exists>
</file>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>add-classpath-descriptor</id>
<phase>prepare-package</phase>
<goals>
<goal>build-classpath</goal>
</goals>
<configuration>
<outputFile>${project.build.outputDirectory}/${project.artifactId}.classpath</outputFile>
<prefix>$HDDS_LIB_JARS_DIR</prefix>
<outputFilterFile>true</outputFilterFile>
<includeScope>runtime</includeScope>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
2 changes: 1 addition & 1 deletion hadoop-hdds/rocks-native/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -385,7 +385,7 @@
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-jars</id>
<id>copy-dependencies</id>
<phase>process-sources</phase>
<goals>
<goal>copy-dependencies</goal>
Expand Down
1 change: 1 addition & 0 deletions hadoop-hdds/server-scm/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<packaging>jar</packaging>

<properties>
<classpath.skip>false</classpath.skip>
</properties>

<dependencies>
Expand Down
1 change: 1 addition & 0 deletions hadoop-ozone/csi/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<packaging>jar</packaging>

<properties>
<classpath.skip>false</classpath.skip>
<maven.test.skip>true</maven.test.skip> <!-- no tests in this module so far -->
</properties>

Expand Down
1 change: 1 addition & 0 deletions hadoop-ozone/datanode/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
<version>2.0.0-SNAPSHOT</version>

<properties>
<classpath.skip>false</classpath.skip>
<maven.test.skip>true</maven.test.skip> <!-- no tests in this module so far -->
<spotbugs.skip>true</spotbugs.skip>
</properties>
Expand Down
4 changes: 2 additions & 2 deletions hadoop-ozone/dev-support/checks/dependency.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ cp ${src_dir}/current.txt "$REPORT_DIR"/

#implementation of sort cli is not exactly the same everywhere. It's better to sort with the same command locally
(diff -uw \
<(sort ${src_dir}/jar-report.txt) \
<(sort ${src_dir}/current.txt) \
<(sort -u ${src_dir}/jar-report.txt) \
<(sort -u ${src_dir}/current.txt) \
|| true) \
> "$REPORT_FILE"

Expand Down
2 changes: 1 addition & 1 deletion hadoop-ozone/dev-support/checks/license.exceptions
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
# This file lists dependencies with acceptable license that
# license-maven-plugin cannot find, or finds with unexpected license.

com.google.re2j:re2j:1.7 BSD 3-Clause
com.google.re2j:re2j:1.1 BSD 3-Clause
javax.servlet:servlet-api:2.5 CDDL 1.1
javax.servlet.jsp:jsp-api:2.1 CDDL 1.1
org.codehaus.jettison:jettison:1.1 Apache License 2.0
10 changes: 10 additions & 0 deletions hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ run cp -p "${ROOT}/HISTORY.md" .
run cp -p "${ROOT}/SECURITY.md" .
run cp -p "${ROOT}/CONTRIBUTING.md" .

run mkdir -p ./share/ozone/classpath
run mkdir -p ./share/ozone/lib
run mkdir -p ./share/ozone/web
run mkdir -p ./bin
run mkdir -p ./sbin
Expand Down Expand Up @@ -129,6 +131,14 @@ run cp -p -r "${ROOT}/hadoop-ozone/dist/target/Dockerfile" .
#Copy pre-generated keytabs
run cp -p -R "${ROOT}/hadoop-ozone/dist/src/main/keytabs" compose/_keytabs

for file in $(find "${ROOT}" -path '*/target/classes/*.classpath' | sort); do
cp -n -p -v "$file" share/ozone/classpath/
done

for file in $(find "${ROOT}" -path '*/share/ozone/lib/*jar' | sort); do
cp -n -p -v "$file" share/ozone/lib/
done

#workaround for https://issues.apache.org/jira/browse/MRESOURCES-236
find ./compose -name "*.sh" -exec chmod 755 {} \;
find ./kubernetes -name "*.sh" -exec chmod 755 {} \;
37 changes: 1 addition & 36 deletions hadoop-ozone/dist/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,24 +71,7 @@
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-classpath-files</id>
<phase>prepare-package</phase>
<goals>
<goal>unpack-dependencies</goal>
</goals>
<configuration>
<outputDirectory>
target/ozone-${ozone.version}/share/ozone/classpath
</outputDirectory>
<includes>*.classpath</includes>
<includeArtifactIds>
hdds-server-scm,ozone-common,ozone-csi,ozone-datanode,ozone-httpfsgateway,
ozone-insight,ozone-manager,ozone-recon,ozone-s3gateway,ozone-tools,hdds-rocks-native,ozone-s3-secret-store
</includeArtifactIds>
</configuration>
</execution>
<execution>
<id>copy-jars</id>
<id>copy-omitted-jars</id>
<phase>prepare-package</phase>
<goals>
<goal>copy-dependencies</goal>
Expand All @@ -98,24 +81,6 @@
<includeScope>runtime</includeScope>
</configuration>
</execution>
<execution>
<id>copy-omitted-jars</id>
<phase>prepare-package</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<outputDirectory>target/ozone-${ozone.version}/share/ozone/lib
</outputDirectory>
<artifactItems>
<artifactItem>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>${grpc.protobuf-compile.version}</version>
</artifactItem>
</artifactItems>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
Expand Down
1 change: 1 addition & 0 deletions hadoop-ozone/httpfsgateway/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
<description>Apache Ozone HttpFS</description>

<properties>
<classpath.skip>false</classpath.skip>
<httpfs.source.repository>REPO NOT AVAIL</httpfs.source.repository>
<httpfs.source.revision>REVISION NOT AVAIL</httpfs.source.revision>
<maven.build.timestamp.format>yyyy-MM-dd'T'HH:mm:ssZ</maven.build.timestamp.format>
Expand Down
2 changes: 2 additions & 0 deletions hadoop-ozone/insight/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<description>Apache Ozone Insight Tool</description>
<name>Apache Ozone Insight Tool</name>
<packaging>jar</packaging>

<properties>
<classpath.skip>false</classpath.skip>
</properties>

<dependencies>
Expand Down
20 changes: 20 additions & 0 deletions hadoop-ozone/integration-test/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,26 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<dependency>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-server-scm</artifactId>
<exclusions>
<!--
Dependency convergence error for com.fasterxml.woodstox:woodstox-core

+-org.apache.ozone:ozone-integration-test:jar:1.5.0-SNAPSHOT
+-org.apache.ozone:hdds-server-scm:jar:1.5.0-SNAPSHOT:compile
+-org.apache.ozone:hdds-hadoop-dependency-server:jar:1.5.0-SNAPSHOT:compile
+-org.apache.hadoop:hadoop-common:jar:3.3.6:compile
+-com.fasterxml.woodstox:woodstox-core:jar:5.4.0:compile

+-org.apache.ozone:ozone-integration-test:jar:1.5.0-SNAPSHOT
+-org.apache.ozone:ozone-s3gateway:jar:1.5.0-SNAPSHOT:compile
+-com.fasterxml.jackson.dataformat:jackson-dataformat-xml:jar:2.16.2:compile
+-com.fasterxml.woodstox:woodstox-core:jar:6.5.1:compile
-->
<exclusion>
<groupId>com.fasterxml.woodstox</groupId>
<artifactId>woodstox-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.ozone</groupId>
Expand Down
1 change: 1 addition & 0 deletions hadoop-ozone/ozone-manager/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<packaging>jar</packaging>

<properties>
<classpath.skip>false</classpath.skip>
</properties>

<dependencies>
Expand Down
50 changes: 19 additions & 31 deletions hadoop-ozone/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -315,6 +315,25 @@
<rules>
<DependencyConvergence>
<uniqueVersions>true</uniqueVersions>
<excludes>
<!--
Dependency convergence error for org.codehaus.woodstox:stax2-api

+-org.apache.ozone:ozone-common:jar:1.5.0-SNAPSHOT
+-org.apache.ozone:hdds-common:jar:1.5.0-SNAPSHOT:compile
+-org.apache.ozone:hdds-hadoop-dependency-client:jar:1.5.0-SNAPSHOT:compile
+-org.apache.hadoop:hadoop-common:jar:3.3.6:compile
+-org.codehaus.woodstox:stax2-api:jar:4.2.1:compile

+-org.apache.ozone:ozone-common:jar:1.5.0-SNAPSHOT
+-org.apache.ozone:hdds-common:jar:1.5.0-SNAPSHOT:compile
+-org.apache.ozone:hdds-hadoop-dependency-client:jar:1.5.0-SNAPSHOT:compile
+-org.apache.hadoop:hadoop-common:jar:3.3.6:compile
+-com.fasterxml.woodstox:woodstox-core:jar:5.4.0:compile
+-org.codehaus.woodstox:stax2-api:jar:4.2:compile
-->
<exclude>org.codehaus.woodstox:stax2-api</exclude>
</excludes>
</DependencyConvergence>
</rules>
</configuration>
Expand Down Expand Up @@ -455,36 +474,5 @@
</plugins>
</build>
</profile>
<profile>
<id>add-classpath-descriptor</id>
<activation>
<file>
<exists>src/main/java</exists>
</file>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>add-classpath-descriptor</id>
<phase>prepare-package</phase>
<goals>
<goal>build-classpath</goal>
</goals>
<configuration>
<outputFile>${project.build.outputDirectory}/${project.artifactId}.classpath</outputFile>
<prefix>$HDDS_LIB_JARS_DIR</prefix>
<outputFilterFile>true</outputFilterFile>
<includeScope>runtime</includeScope>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
1 change: 1 addition & 0 deletions hadoop-ozone/recon/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>ozone-recon</artifactId>
<properties>
<classpath.skip>false</classpath.skip>
<pnpm.version>8.15.7</pnpm.version>
</properties>
<build>
Expand Down
19 changes: 19 additions & 0 deletions hadoop-ozone/s3gateway/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
<packaging>jar</packaging>
<version>2.0.0-SNAPSHOT</version>
<properties>
<classpath.skip>false</classpath.skip>
<file.encoding>UTF-8</file.encoding>
<downloadSources>true</downloadSources>
</properties>
Expand Down Expand Up @@ -226,6 +227,24 @@
<dependency>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-hadoop-dependency-server</artifactId>
<exclusions>
<!--
Dependency convergence error for com.fasterxml.woodstox:woodstox-core

+-org.apache.ozone:ozone-s3gateway:jar:1.5.0-SNAPSHOT
+-com.fasterxml.jackson.dataformat:jackson-dataformat-xml:jar:2.16.2:compile
+-com.fasterxml.woodstox:woodstox-core:jar:6.5.1:compile

+-org.apache.ozone:ozone-s3gateway:jar:1.5.0-SNAPSHOT
+-org.apache.ozone:hdds-hadoop-dependency-server:jar:1.5.0-SNAPSHOT:compile
+-org.apache.hadoop:hadoop-common:jar:3.3.6:compile
+-com.fasterxml.woodstox:woodstox-core:jar:5.4.0:compile
-->
<exclusion>
<groupId>com.fasterxml.woodstox</groupId>
<artifactId>woodstox-core</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.ozone</groupId>
Expand Down
Loading