diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f81c1ad8b1..1148df52c9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,7 +27,7 @@ jobs: build: strategy: matrix: - java-version: [8, 11, 17] + java-version: [17] os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} steps: diff --git a/Jenkinsfile b/Jenkinsfile index 4ce236598d..a7fcecdc47 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -117,12 +117,6 @@ pipeline { YETUS_ARGS+=("--proclimit=5500") YETUS_ARGS+=("--dockermemlimit=20g") - # -1 findbugs issues that show up prior to the patch being applied - # YETUS_ARGS+=("--findbugs-strict-precheck") - - # rsync these files back into the archive dir - YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,findbugsXml.xml") - # URL for user-side presentation in reports and such to our artifacts # (needs to match the archive bits below) YETUS_ARGS+=("--build-url-artifacts=artifact/out") @@ -152,9 +146,9 @@ pipeline { # use emoji vote so it is easier to find the broken line YETUS_ARGS+=("--github-use-emoji-vote") - # test with Java 8 and 11 - YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64") - YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64") + # test with Java 17 + YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-17-openjdk-amd64") + YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-17-openjdk-amd64") YETUS_ARGS+=("--multijdktests=compile") YETUS_ARGS+=("--debug") diff --git a/build-tools/docker/Dockerfile b/build-tools/docker/Dockerfile index 5b681067ae..c292bfae47 100644 --- a/build-tools/docker/Dockerfile +++ b/build-tools/docker/Dockerfile @@ -165,29 +165,17 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] #### #### -# OpenJDK 8 -#### -# hadolint ignore=DL3008 -RUN apt-get -q update && apt-get -q install --no-install-recommends -y openjdk-8-jdk-headless \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* +# OpenJDK 17 +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y openjdk-17-jdk && \ + apt-get clean && rm -rf /var/lib/apt/lists/* -#### -# OpenJDK 11 (but keeps default to JDK8) -# NOTE: This default only works when Apache Yetus is launched -# _in_ the container and not outside of it! -#### -# hadolint ignore=DL3008 -RUN apt-get -q update && apt-get -q install --no-install-recommends -y default-jre-headless openjdk-11-jdk-headless \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && update-java-alternatives -s java-1.8.0-openjdk-amd64 || : -# since update alternatives might fail on executables that we don't really need (e.g. appletviewer) -# and return with exit code <0 (actually: 2), we can simply do a sanity check if the version is -# as expected for "java" executable after the update and go on -RUN java -version 2>&1 | grep "1.8.0" && rm -f /usr/lib/jvm/default-java \ - && ln -s java-8-openjdk-amd64 /usr/lib/jvm/default-java -ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 +# Set JAVA_HOME and PATH environment variables +ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64 +ENV PATH="${JAVA_HOME}/bin:${PATH}" + +# Set the default Java version using update-alternatives +RUN update-alternatives --install /usr/bin/java java /usr/lib/jvm/java-17-openjdk-amd64/bin/java 1 ###### # Install findbugs diff --git a/pom.xml b/pom.xml index 0ab151d9d0..7caa2c1f08 100644 --- a/pom.xml +++ b/pom.xml @@ -41,8 +41,8 @@ - 1.8 - 1.8 + 17 + 17 true 3.0.0-M4 false @@ -101,6 +101,7 @@ 1.1.10.4 ${project.build.directory}/tmp 1.7.9 + 15.4 true @@ -156,6 +157,11 @@ + + org.openjdk.nashorn + nashorn-core + ${nashorn.version} + org.apache.tez hadoop-shim @@ -926,7 +932,13 @@ 1 false 900 - -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError + -Xmx4096m -XX:+HeapDumpOnOutOfMemoryError + + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens java.base/java.io=ALL-UNNAMED + + true ${java.home} 4 @@ -1122,6 +1134,7 @@ org.codehaus.mojo findbugs-maven-plugin + true ${basedir}/findbugs-exclude.xml @@ -1237,6 +1250,7 @@ + true ${basedir}/findbugs-exclude.xml @@ -1248,6 +1262,7 @@ org.codehaus.mojo findbugs-maven-plugin + true xml html diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java index 8862f4b7d6..1cfb8cd4fe 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java @@ -396,8 +396,17 @@ public TezConfiguration(boolean loadDefaults) { @ConfigurationProperty public static final String TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS = TEZ_AM_PREFIX + "launch.cluster-default.cmd-opts"; - public static final String TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT = + public static final String TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT; + + static { + if (TezCommonUtils.getJavaVersion() >= 9) { + TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT = + "-server -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN --add-opens java.base/java.lang=ALL-UNNAMED"; + } else { + TEZ_AM_LAUNCH_CLUSTER_DEFAULT_CMD_OPTS_DEFAULT = "-server -Djava.net.preferIPv4Stack=true -Dhadoop.metrics.log.level=WARN"; + } + } /** * String value. Command line options provided during the launch of the Tez diff --git a/tez-dag/pom.xml b/tez-dag/pom.xml index 779f7ed82f..b135cabd01 100644 --- a/tez-dag/pom.xml +++ b/tez-dag/pom.xml @@ -36,6 +36,11 @@ tez-dag + + org.openjdk.nashorn + nashorn-core + ${nashorn.version} + org.apache.tez hadoop-shim @@ -205,6 +210,11 @@ org.apache.maven.plugins maven-surefire-plugin + + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens java.base/java.io=ALL-UNNAMED + ${test.log.dir} diff --git a/tez-ext-service-tests/pom.xml b/tez-ext-service-tests/pom.xml index 532588a9ce..8e88888e86 100644 --- a/tez-ext-service-tests/pom.xml +++ b/tez-ext-service-tests/pom.xml @@ -30,6 +30,10 @@ + + org.openjdk.nashorn + nashorn-core + io.netty netty-all @@ -112,7 +116,6 @@ hadoop-mapreduce-client-shuffle test - @@ -161,6 +164,11 @@ org.apache.maven.plugins maven-surefire-plugin + + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens java.base/java.io=ALL-UNNAMED + ${test.log.dir} diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/ExternalTezServiceTestHelper.java b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/ExternalTezServiceTestHelper.java index c3831223c7..6ce1cdf957 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/ExternalTezServiceTestHelper.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/ExternalTezServiceTestHelper.java @@ -78,10 +78,17 @@ public ExternalTezServiceTestHelper(String testRootDir) throws for (Map.Entry entry : tezCluster.getConfig()) { clusterConf.set(entry.getKey(), entry.getValue()); } + long jvmMax = Runtime.getRuntime().maxMemory(); + long availableMemory; + if ((jvmMax * 0.5d) >= Integer.MAX_VALUE) { + availableMemory = Integer.MAX_VALUE; + } else { + availableMemory = (long) (jvmMax * 0.5d); + } tezTestServiceCluster = MiniTezTestServiceCluster - .create(TestExternalTezServices.class.getSimpleName(), 3, ((long) (jvmMax * 0.5d)), 1); + .create(TestExternalTezServices.class.getSimpleName(), 3, availableMemory, 1); tezTestServiceCluster.init(clusterConf); tezTestServiceCluster.start(); LOG.info("MiniTezTestServer started"); diff --git a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExtServicesWithLocalMode.java b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExtServicesWithLocalMode.java index c0bfe76bd4..2dcd216610 100644 --- a/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExtServicesWithLocalMode.java +++ b/tez-ext-service-tests/src/test/java/org/apache/tez/tests/TestExtServicesWithLocalMode.java @@ -83,8 +83,14 @@ public static void setup() throws Exception { localFs = FileSystem.getLocal(clusterConf).getRaw(); long jvmMax = Runtime.getRuntime().maxMemory(); + long availableMemory; + if ((jvmMax * 0.5d) >= Integer.MAX_VALUE) { + availableMemory = Integer.MAX_VALUE; + } else { + availableMemory = (long) (jvmMax * 0.5d); + } tezTestServiceCluster = MiniTezTestServiceCluster - .create(TestExternalTezServices.class.getSimpleName(), 3, ((long) (jvmMax * 0.5d)), 1); + .create(TestExternalTezServices.class.getSimpleName(), 3, availableMemory, 1); tezTestServiceCluster.init(clusterConf); tezTestServiceCluster.start(); LOG.info("MiniTezTestServer started"); diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestSimpleFetchedInputAllocator.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestSimpleFetchedInputAllocator.java index 01faa5df7a..3b826439fe 100644 --- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestSimpleFetchedInputAllocator.java +++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestSimpleFetchedInputAllocator.java @@ -66,7 +66,7 @@ public void testInMemAllocation() throws IOException { FetchedInput fi2 = inputManager.allocate(requestSize, compressedSize, new InputAttemptIdentifier(2, 1)); - assertEquals(FetchedInput.Type.MEMORY, fi2.getType()); + assertEquals(FetchedInput.Type.DISK, fi2.getType()); // Over limit by this point. Next reserve should give back a DISK allocation diff --git a/tez-tests/pom.xml b/tez-tests/pom.xml index 2bf8ca38ee..7a003902ce 100644 --- a/tez-tests/pom.xml +++ b/tez-tests/pom.xml @@ -29,6 +29,10 @@ + + org.openjdk.nashorn + nashorn-core + org.apache.tez tez-api @@ -160,6 +164,11 @@ org.apache.maven.plugins maven-surefire-plugin + + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens java.base/java.io=ALL-UNNAMED + ${test.log.dir}