diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index f81c1ad8b1..1148df52c9 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -27,7 +27,7 @@ jobs:
build:
strategy:
matrix:
- java-version: [8, 11, 17]
+ java-version: [17]
os: [ubuntu-latest, macos-latest]
runs-on: ${{ matrix.os }}
steps:
diff --git a/Jenkinsfile b/Jenkinsfile
index 4ce236598d..a7fcecdc47 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -117,12 +117,6 @@ pipeline {
YETUS_ARGS+=("--proclimit=5500")
YETUS_ARGS+=("--dockermemlimit=20g")
- # -1 findbugs issues that show up prior to the patch being applied
- # YETUS_ARGS+=("--findbugs-strict-precheck")
-
- # rsync these files back into the archive dir
- YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,findbugsXml.xml")
-
# URL for user-side presentation in reports and such to our artifacts
# (needs to match the archive bits below)
YETUS_ARGS+=("--build-url-artifacts=artifact/out")
@@ -152,9 +146,9 @@ pipeline {
# use emoji vote so it is easier to find the broken line
YETUS_ARGS+=("--github-use-emoji-vote")
- # test with Java 8 and 11
- YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64")
- YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64")
+ # test with Java 17
+ YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-17-openjdk-amd64")
+ YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-17-openjdk-amd64")
YETUS_ARGS+=("--multijdktests=compile")
YETUS_ARGS+=("--debug")
diff --git a/build-tools/docker/Dockerfile b/build-tools/docker/Dockerfile
index 5b681067ae..c292bfae47 100644
--- a/build-tools/docker/Dockerfile
+++ b/build-tools/docker/Dockerfile
@@ -165,29 +165,17 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
####
####
-# OpenJDK 8
-####
-# hadolint ignore=DL3008
-RUN apt-get -q update && apt-get -q install --no-install-recommends -y openjdk-8-jdk-headless \
- && apt-get clean \
- && rm -rf /var/lib/apt/lists/*
+# OpenJDK 17
+RUN apt-get update && \
+ DEBIAN_FRONTEND=noninteractive apt-get install -y openjdk-17-jdk && \
+ apt-get clean && rm -rf /var/lib/apt/lists/*
-####
-# OpenJDK 11 (but keeps default to JDK8)
-# NOTE: This default only works when Apache Yetus is launched
-# _in_ the container and not outside of it!
-####
-# hadolint ignore=DL3008
-RUN apt-get -q update && apt-get -q install --no-install-recommends -y default-jre-headless openjdk-11-jdk-headless \
- && apt-get clean \
- && rm -rf /var/lib/apt/lists/* \
- && update-java-alternatives -s java-1.8.0-openjdk-amd64 || :
-# since update alternatives might fail on executables that we don't really need (e.g. appletviewer)
-# and return with exit code <0 (actually: 2), we can simply do a sanity check if the version is
-# as expected for "java" executable after the update and go on
-RUN java -version 2>&1 | grep "1.8.0" && rm -f /usr/lib/jvm/default-java \
- && ln -s java-8-openjdk-amd64 /usr/lib/jvm/default-java
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
+# Set JAVA_HOME and PATH environment variables
+ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64
+ENV PATH="${JAVA_HOME}/bin:${PATH}"
+
+# Set the default Java version using update-alternatives
+RUN update-alternatives --install /usr/bin/java java /usr/lib/jvm/java-17-openjdk-amd64/bin/java 1
######
# Install findbugs
diff --git a/pom.xml b/pom.xml
index 0ab151d9d0..7caa2c1f08 100644
--- a/pom.xml
+++ b/pom.xml
@@ -41,8 +41,8 @@
- 1.8
- 1.8
+ 17
+ 17
true
3.0.0-M4
false
@@ -101,6 +101,7 @@
1.1.10.4
${project.build.directory}/tmp
1.7.9
+ 15.4
true
@@ -156,6 +157,11 @@
+
+ org.openjdk.nashorn
+ nashorn-core
+ ${nashorn.version}
+
org.apache.tez
hadoop-shim
@@ -926,7 +932,13 @@
1
false
900
- -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError
+ -Xmx4096m -XX:+HeapDumpOnOutOfMemoryError
+
+ --add-opens=java.base/java.lang=ALL-UNNAMED
+ --add-opens=java.base/java.util=ALL-UNNAMED
+ --add-opens java.base/java.io=ALL-UNNAMED
+
+ true
${java.home}
4
@@ -1122,6 +1134,7 @@
org.codehaus.mojo
findbugs-maven-plugin
+ true
${basedir}/findbugs-exclude.xml
@@ -1237,6 +1250,7 @@
+ true
${basedir}/findbugs-exclude.xml
@@ -1248,6 +1262,7 @@
org.codehaus.mojo
findbugs-maven-plugin
+ true
xml
html
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 8862f4b7d6..1cfb8cd4fe 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -396,8 +396,17 @@ public TezConfiguration(boolean loadDefaults) {
@ConfigurationProperty
public static final String TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS =
TEZ_AM_PREFIX + "launch.cluster-default.cmd-opts";
- public static final String TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT =
+ public static final String TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT;
+
+ static {
+ if (TezCommonUtils.getJavaVersion() >= 9) {
+ TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT =
+ "-server -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN --add-opens java.base/java.lang=ALL-UNNAMED";
+ } else {
+ TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT =
"-server -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN";
+ }
+ }
/**
* String value. Command line options provided during the launch of the Tez
diff --git a/tez-dag/pom.xml b/tez-dag/pom.xml
index 779f7ed82f..b135cabd01 100644
--- a/tez-dag/pom.xml
+++ b/tez-dag/pom.xml
@@ -36,6 +36,11 @@
tez-dag
+
+ org.openjdk.nashorn
+ nashorn-core
+ ${nashorn.version}
+
org.apache.tez
hadoop-shim
@@ -205,6 +210,11 @@
org.apache.maven.plugins
maven-surefire-plugin
+
+ --add-opens=java.base/java.lang=ALL-UNNAMED
+ --add-opens=java.base/java.util=ALL-UNNAMED
+ --add-opens java.base/java.io=ALL-UNNAMED
+
${test.log.dir}
diff --git a/tez-ext-service-tests/pom.xml b/tez-ext-service-tests/pom.xml
index 532588a9ce..8e88888e86 100644
--- a/tez-ext-service-tests/pom.xml
+++ b/tez-ext-service-tests/pom.xml
@@ -30,6 +30,10 @@
+
+ org.openjdk.nashorn
+ nashorn-core
+
io.netty
netty-all
@@ -112,7 +116,6 @@
hadoop-mapreduce-client-shuffle
test
-
@@ -161,6 +164,11 @@
org.apache.maven.plugins
maven-surefire-plugin
+
+ --add-opens=java.base/java.lang=ALL-UNNAMED
+ --add-opens=java.base/java.util=ALL-UNNAMED
+ --add-opens java.base/java.io=ALL-UNNAMED
+
${test.log.dir}
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/ExternalTezServiceTestHelper.java b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/ExternalTezServiceTestHelper.java
index c3831223c7..6ce1cdf957 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/ExternalTezServiceTestHelper.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/ExternalTezServiceTestHelper.java
@@ -78,10 +78,17 @@ public ExternalTezServiceTestHelper(String testRootDir) throws
for (Map.Entry entry : tezCluster.getConfig()) {
clusterConf.set(entry.getKey(), entry.getValue());
}
+
long jvmMax = Runtime.getRuntime().maxMemory();
+ long availableMemory;
+ if ((jvmMax * 0.5d) >= Integer.MAX_VALUE) {
+ availableMemory = Integer.MAX_VALUE;
+ } else {
+ availableMemory = (long) (jvmMax * 0.5d);
+ }
tezTestServiceCluster = MiniTezTestServiceCluster
- .create(TestExternalTezServices.class.getSimpleName(), 3, ((long) (jvmMax * 0.5d)), 1);
+ .create(TestExternalTezServices.class.getSimpleName(), 3, availableMemory, 1);
tezTestServiceCluster.init(clusterConf);
tezTestServiceCluster.start();
LOG.info("MiniTezTestServer started");
diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExtServicesWithLocalMode.java b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExtServicesWithLocalMode.java
index c0bfe76bd4..2dcd216610 100644
--- a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExtServicesWithLocalMode.java
+++ b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExtServicesWithLocalMode.java
@@ -83,8 +83,14 @@ public static void setup() throws Exception {
localFs = FileSystem.getLocal(clusterConf).getRaw();
long jvmMax = Runtime.getRuntime().maxMemory();
+ long availableMemory;
+ if ((jvmMax * 0.5d) >= Integer.MAX_VALUE) {
+ availableMemory = Integer.MAX_VALUE;
+ } else {
+ availableMemory = (long) (jvmMax * 0.5d);
+ }
tezTestServiceCluster = MiniTezTestServiceCluster
- .create(TestExternalTezServices.class.getSimpleName(), 3, ((long) (jvmMax * 0.5d)), 1);
+ .create(TestExternalTezServices.class.getSimpleName(), 3, availableMemory, 1);
tezTestServiceCluster.init(clusterConf);
tezTestServiceCluster.start();
LOG.info("MiniTezTestServer started");
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestSimpleFetchedInputAllocator.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestSimpleFetchedInputAllocator.java
index 01faa5df7a..3b826439fe 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestSimpleFetchedInputAllocator.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestSimpleFetchedInputAllocator.java
@@ -66,7 +66,7 @@ public void testInMemAllocation() throws IOException {
FetchedInput fi2 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(2, 1));
- assertEquals(FetchedInput.Type.MEMORY, fi2.getType());
+ assertEquals(FetchedInput.Type.DISK, fi2.getType());
// Over limit by this point. Next reserve should give back a DISK allocation
diff --git a/tez-tests/pom.xml b/tez-tests/pom.xml
index 2bf8ca38ee..7a003902ce 100644
--- a/tez-tests/pom.xml
+++ b/tez-tests/pom.xml
@@ -29,6 +29,10 @@
+
+ org.openjdk.nashorn
+ nashorn-core
+
org.apache.tez
tez-api
@@ -160,6 +164,11 @@
org.apache.maven.plugins
maven-surefire-plugin
+
+ --add-opens=java.base/java.lang=ALL-UNNAMED
+ --add-opens=java.base/java.util=ALL-UNNAMED
+ --add-opens java.base/java.io=ALL-UNNAMED
+
${test.log.dir}