diff --git a/.github/bin/retry b/.github/bin/retry new file mode 100755 index 0000000000000..9e9bd9da4d626 --- /dev/null +++ b/.github/bin/retry @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -euo pipefail + +x() { + echo "+ $*" >&2 + "$@" +} + +max_retry_time_seconds=$(( 30 * 60 )) +retry_delay_seconds=10 + +END=$(( $(date +%s) + ${max_retry_time_seconds} )) + +while (( $(date +%s) < $END )); do + x "$@" && exit 0 + sleep "${retry_delay_seconds}" +done + +echo "$0: retrying [$*] timed out" >&2 +exit 1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000000..d91009fa09a52 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,201 @@ +name: ci + +on: + pull_request: + +env: + # An envar that signals to tests we are executing in the CI environment + CONTINUOUS_INTEGRATION: true + MAVEN_OPTS: "-Xmx1024M -XX:+ExitOnOutOfMemoryError" + MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" + MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" + MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" + RETRY: .github/bin/retry + +jobs: + maven-checks: + runs-on: ubuntu-latest + strategy: + fail-fast: false + timeout-minutes: 45 + steps: + - name: Free Disk Space + run: | + df -h + sudo apt-get clean + df -h + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: 8 + - name: Maven Checks + run: | + export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" + $RETRY ./mvnw install -B -V -T C1 -DskipTests -P ci -pl '!:presto-server-rpm' + - name: Test Server RPM + run: | + export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" + $RETRY ./mvnw verify -B -P ci -pl :presto-server-rpm + - name: Clean Maven Output + run: ./mvnw clean -pl '!:presto-server,!:presto-cli' + + web-ui-checks: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v2 + - name: Web UI Checks + run: presto-main/bin/check_webui.sh + + hive-tests: + runs-on: ubuntu-latest + strategy: + fail-fast: false + timeout-minutes: 60 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: 8 + - name: Install Hive Module + run: | + export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" + $RETRY ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl :presto-hive-hadoop2 + - name: Run Hive Tests + run: presto-hive-hadoop2/bin/run_hive_tests.sh + - name: Run Hive S3 Tests + env: + AWS_ACCESS_KEY_ID: ${{ secrets.HIVE_AWS_ACCESSKEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.HIVE_AWS_SECRETKEY }} + S3_BUCKET: "presto-ci-test" + S3_BUCKET_ENDPOINT: "s3.us-east-2.amazonaws.com" + run: | + if [ "${AWS_ACCESS_KEY_ID}" != "" ]; then + presto-hive-hadoop2/bin/run_hive_s3_tests.sh + fi + - name: Run Hive Glue Tests + env: + AWS_ACCESS_KEY_ID: ${{ secrets.HIVE_AWS_ACCESSKEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.HIVE_AWS_SECRETKEY }} + run: | + if [ "${HIVE_AWS_ACCESS_KEY_ID}" != "" ]; then + ./mvnw test ${MAVEN_TEST} -pl :presto-hive -P test-hive-glue + fi + + test-other-modules: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: 8 + - name: Maven Install + run: | + export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" + $RETRY ./mvnw install ${MAVEN_FAST_INSTALL} -pl '!:presto-docs,!:presto-server,!:presto-server-rpm' + - name: Maven Tests + run: | + ./mvnw test ${MAVEN_TEST} -pl ' + !presto-tests, + !presto-raptor, + !presto-accumulo, + !presto-cassandra, + !presto-hive, + !presto-kudu, + !presto-docs, + !presto-server, + !presto-server-rpm, + !presto-main, + !presto-mongodb, + !presto-spark-package, + !presto-spark-launcher, + !presto-spark-testing, + !presto-spark-base, + !presto-redis, + !presto-elasticsearch, + !presto-orc, + !presto-thrift-connector' + + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + modules: + - ":presto-docs" + - ":presto-tests -P presto-tests-execution-memory" + - ":presto-tests -P presto-tests-general" + - ":presto-tests -P ci-only-distributed-non-hash-gen" + - ":presto-tests -P ci-only-tpch-distributed-queries" + - ":presto-tests -P ci-only-local-queries" + - ":presto-tests -P ci-only-distributed-queries" + - ":presto-tests -P ci-only-aggregation-queries" + - ":presto-tests -P ci-only-plan-determinism" + - ":presto-raptor" + - ":presto-accumulo" + - ":presto-cassandra -P test-cassandra-integration-smoke-test" + - ":presto-hive" + - ":presto-hive -P test-hive-materialized-queries" + - ":presto-hive -P test-hive-materialized-aggregations" + - ":presto-hive -P test-hive-recoverable-execution" + - ":presto-hive -P test-hive-pushdown-filter-queries-basic" + - ":presto-hive -P test-hive-pushdown-filter-queries-advanced" + - ":presto-hive -P test-hive-repartitioning" + - ":presto-hive -P test-hive-parquet" + - ":presto-main" + - ":presto-mongodb -P test-mongo-distributed-queries" + - ":presto-redis -P test-redis-integration-smoke-test" + - ":presto-elasticsearch" + - ":presto-orc" + - ":presto-thrift-connector" + - ":presto-spark-base" + timeout-minutes: 80 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: 8 + - name: Maven Install + run: | + export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" + $RETRY ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl $(echo '${{ matrix.modules }}' | cut -d' ' -f1) + - name: Maven Tests + run: ./mvnw test ${MAVEN_TEST} -pl ${{ matrix.modules }} + + kudu: + runs-on: ubuntu-latest + strategy: + fail-fast: false + timeout-minutes: 60 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: 8 + - name: Maven Install + run: | + export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" + $RETRY ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl presto-kudu + - name: Kudu Tests + run: | + presto-kudu/bin/run_kudu_tests.sh 3 null + presto-kudu/bin/run_kudu_tests.sh 1 "" + presto-kudu/bin/run_kudu_tests.sh 1 presto:: + + spark-integration: + runs-on: ubuntu-latest + strategy: + fail-fast: false + timeout-minutes: 60 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: 8 + - name: Maven Install + run: | + export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" + $RETRY ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl presto-kudu + - name: Maven Tests + run: $RETRY ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl presto-spark-launcher,presto-spark-package,presto-spark-testing -P test-presto-spark-integration-smoke-test diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml new file mode 100644 index 0000000000000..0021fe31e5acc --- /dev/null +++ b/.github/workflows/cleanup.yml @@ -0,0 +1,14 @@ +name: cleanup + +on: + schedule: + - cron: '*/5 * * * *' + +jobs: + cancel: + runs-on: ubuntu-latest + steps: + - uses: n1hility/cancel-previous-runs@2e3c1893986568a2197c41957b9c809cbcf1a61e + with: + token: ${{ github.token }} + workflow: ci.yml diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties index c315043703752..be4fea70726b6 100644 --- a/.mvn/wrapper/maven-wrapper.properties +++ b/.mvn/wrapper/maven-wrapper.properties @@ -1 +1 @@ -distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.5.0/apache-maven-3.5.0-bin.zip +distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip diff --git a/pom.xml b/pom.xml index fb65e5cc3653d..125fa4aa374d2 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ com.facebook.airlift airbase - 100 + 101 com.facebook.presto @@ -54,7 +54,7 @@ 3.4.0 19.3.0.0 1.32 - 2.10.5 + 2.10.6 1.50 6.10 3.8.0 diff --git a/presto-common/src/main/resources/com/facebook/presto/common/type/zone-index.properties b/presto-common/src/main/resources/com/facebook/presto/common/type/zone-index.properties index 764ec422b5098..a73dadcee7b54 100644 --- a/presto-common/src/main/resources/com/facebook/presto/common/type/zone-index.properties +++ b/presto-common/src/main/resources/com/facebook/presto/common/type/zone-index.properties @@ -2236,3 +2236,4 @@ 2227 Asia/Famagusta 2228 Europe/Saratov 2229 Asia/Qostanay +2230 America/Nuuk diff --git a/presto-common/src/test/java/com/facebook/presto/common/type/TestTimeZoneKey.java b/presto-common/src/test/java/com/facebook/presto/common/type/TestTimeZoneKey.java index 0d954e07d5701..1b7afe015722c 100644 --- a/presto-common/src/test/java/com/facebook/presto/common/type/TestTimeZoneKey.java +++ b/presto-common/src/test/java/com/facebook/presto/common/type/TestTimeZoneKey.java @@ -213,7 +213,7 @@ public int compare(TimeZoneKey left, TimeZoneKey right) hasher.putString(timeZoneKey.getId(), StandardCharsets.UTF_8); } // Zone file should not (normally) be changed, so let's make this more difficult - assertEquals(hasher.hash().asLong(), -972834036790299986L, "zone-index.properties file contents changed!"); + assertEquals(hasher.hash().asLong(), -3809591333307967388L, "zone-index.properties file contents changed!"); } public void assertTimeZoneNotSupported(String zoneId) diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java index cc1fe0edb871f..1847bd5e90799 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java @@ -307,7 +307,7 @@ public HiveClientConfig setRecursiveDirWalkerEnabled(boolean recursiveDirWalkerE public DateTimeZone getDateTimeZone() { - return DateTimeZone.forTimeZone(TimeZone.getTimeZone(timeZone)); + return DateTimeZone.forID(timeZone); } @NotNull diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java index ca3a17ff34f32..8e6f4596b647c 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/AbstractTestHiveClient.java @@ -139,6 +139,7 @@ import java.io.IOException; import java.math.BigDecimal; +import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -902,7 +903,7 @@ protected void setupHive(String connectorId, String databaseName, String timeZon "layout", Optional.empty(), false)); - timeZone = DateTimeZone.forTimeZone(TimeZone.getTimeZone(timeZoneId)); + timeZone = DateTimeZone.forTimeZone(TimeZone.getTimeZone(ZoneId.of(timeZoneId))); } protected final void setup(String host, int port, String databaseName, String timeZone) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientConfig.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientConfig.java index 05ba7fc888128..09faeaf7b82fd 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientConfig.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveClientConfig.java @@ -26,6 +26,7 @@ import io.airlift.units.Duration; import org.testng.annotations.Test; +import java.time.ZoneId; import java.util.Map; import java.util.TimeZone; import java.util.concurrent.TimeUnit; @@ -262,7 +263,7 @@ public void testExplicitPropertyMappings() .build(); HiveClientConfig expected = new HiveClientConfig() - .setTimeZone(nonDefaultTimeZone().toTimeZone().getID()) + .setTimeZone(TimeZone.getTimeZone(ZoneId.of(nonDefaultTimeZone().getID())).getID()) .setMaxSplitSize(new DataSize(256, Unit.MEGABYTE)) .setMaxPartitionsPerScan(123) .setMaxOutstandingSplits(10) diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveMetadataFileFormatEncryptionSettings.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveMetadataFileFormatEncryptionSettings.java index 0e4a29711eb82..8974a8efcdf1b 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveMetadataFileFormatEncryptionSettings.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveMetadataFileFormatEncryptionSettings.java @@ -40,6 +40,7 @@ import org.testng.annotations.Test; import java.io.File; +import java.time.ZoneId; import java.util.List; import java.util.Map; import java.util.Optional; @@ -105,7 +106,7 @@ public void setup() metastore, HDFS_ENVIRONMENT, new HivePartitionManager(FUNCTION_AND_TYPE_MANAGER, HIVE_CLIENT_CONFIG), - DateTimeZone.forTimeZone(TimeZone.getTimeZone(HIVE_CLIENT_CONFIG.getTimeZone())), + DateTimeZone.forTimeZone(TimeZone.getTimeZone(ZoneId.of(HIVE_CLIENT_CONFIG.getTimeZone()))), true, false, false, diff --git a/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestPrestoDriver.java b/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestPrestoDriver.java index b96e6e182bf9c..0425fc56657db 100644 --- a/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestPrestoDriver.java +++ b/presto-jdbc/src/test/java/com/facebook/presto/jdbc/TestPrestoDriver.java @@ -63,6 +63,7 @@ import java.sql.Time; import java.sql.Timestamp; import java.sql.Types; +import java.time.ZoneId; import java.util.ArrayList; import java.util.GregorianCalendar; import java.util.List; @@ -110,7 +111,7 @@ public class TestPrestoDriver { private static final DateTimeZone ASIA_ORAL_ZONE = DateTimeZone.forID("Asia/Oral"); - private static final GregorianCalendar ASIA_ORAL_CALENDAR = new GregorianCalendar(ASIA_ORAL_ZONE.toTimeZone()); + private static final GregorianCalendar ASIA_ORAL_CALENDAR = new GregorianCalendar(TimeZone.getTimeZone(ZoneId.of(ASIA_ORAL_ZONE.getID()))); private static final String TEST_CATALOG = "test_catalog"; private TestingPrestoServer server; diff --git a/presto-main/src/test/java/com/facebook/presto/block/TestBlockBuilder.java b/presto-main/src/test/java/com/facebook/presto/block/TestBlockBuilder.java index bd4f3fb9e9d24..cd8b9198b3d11 100644 --- a/presto-main/src/test/java/com/facebook/presto/block/TestBlockBuilder.java +++ b/presto-main/src/test/java/com/facebook/presto/block/TestBlockBuilder.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableList; import io.airlift.slice.Slice; import io.airlift.slice.Slices; +import org.testng.SkipException; import org.testng.annotations.Test; import java.util.Collections; @@ -117,6 +118,9 @@ public void testNewBlockBuilderLike() @Test public void testNewBlockBuilderLikeForLargeBlockBuilder() { + if (true) { + throw new SkipException("https://github.com/prestodb/presto/issues/15653 - Skipped because it OOMs"); + } List channels = ImmutableList.of(VARCHAR); PageBuilder pageBuilder = new PageBuilder(channels); BlockBuilder largeVarcharBlockBuilder = pageBuilder.getBlockBuilder(0); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestDateTimeFunctionsBase.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestDateTimeFunctionsBase.java index f215d65d1dbf6..ffd79a4083116 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestDateTimeFunctionsBase.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestDateTimeFunctionsBase.java @@ -1227,6 +1227,6 @@ private static long millisUtc(OffsetTime offsetTime) private static SqlTimestampWithTimeZone toTimestampWithTimeZone(DateTime dateTime) { - return new SqlTimestampWithTimeZone(dateTime.getMillis(), dateTime.getZone().toTimeZone()); + return new SqlTimestampWithTimeZone(dateTime.getMillis(), getTimeZoneKey(dateTime.getZone().getID())); } } diff --git a/presto-main/src/test/java/com/facebook/presto/type/TestArrayOperators.java b/presto-main/src/test/java/com/facebook/presto/type/TestArrayOperators.java index b91acd35957a6..ceb3b34a753ef 100644 --- a/presto-main/src/test/java/com/facebook/presto/type/TestArrayOperators.java +++ b/presto-main/src/test/java/com/facebook/presto/type/TestArrayOperators.java @@ -43,6 +43,7 @@ import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.time.ZoneId; import java.util.Collections; import java.util.List; import java.util.TimeZone; @@ -1896,7 +1897,7 @@ private static SqlDate sqlDate(String dateString) throws ParseException { DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + dateFormat.setTimeZone(TimeZone.getTimeZone(ZoneId.of("UTC"))); return new SqlDate(toIntExact(TimeUnit.MILLISECONDS.toDays(dateFormat.parse(dateString).getTime()))); } } diff --git a/presto-main/src/test/java/com/facebook/presto/type/TestTimeBase.java b/presto-main/src/test/java/com/facebook/presto/type/TestTimeBase.java index ee274a9a98bcd..a6c06f5835312 100644 --- a/presto-main/src/test/java/com/facebook/presto/type/TestTimeBase.java +++ b/presto-main/src/test/java/com/facebook/presto/type/TestTimeBase.java @@ -25,6 +25,9 @@ import org.joda.time.DateTimeZone; import org.testng.annotations.Test; +import java.time.ZoneId; +import java.util.TimeZone; + import static com.facebook.presto.common.function.OperatorType.INDETERMINATE; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; import static com.facebook.presto.common.type.TimeType.TIME; @@ -140,7 +143,7 @@ public void testCastToTimeWithTimeZone() { assertFunction("cast(TIME '03:04:05.321' as time with time zone)", TIME_WITH_TIME_ZONE, - new SqlTimeWithTimeZone(new DateTime(1970, 1, 1, 3, 4, 5, 321, DATE_TIME_ZONE).getMillis(), DATE_TIME_ZONE.toTimeZone())); + new SqlTimeWithTimeZone(new DateTime(1970, 1, 1, 3, 4, 5, 321, DATE_TIME_ZONE).getMillis(), TimeZone.getTimeZone(ZoneId.of(DATE_TIME_ZONE.getID())))); } @Test @@ -155,7 +158,7 @@ public void testCastToTimeWithTimeZoneWithTZWithRulesChanged() localAssertions.assertFunction( "cast(TIME '03:04:05.321' as time with time zone)", TIME_WITH_TIME_ZONE, - new SqlTimeWithTimeZone(new DateTime(1970, 1, 1, 3, 4, 5, 321, dateTimeZoneThatChangedSince1970).getMillis(), dateTimeZoneThatChangedSince1970.toTimeZone())); + new SqlTimeWithTimeZone(new DateTime(1970, 1, 1, 3, 4, 5, 321, dateTimeZoneThatChangedSince1970).getMillis(), TimeZone.getTimeZone(ZoneId.of(dateTimeZoneThatChangedSince1970.getID())))); } } diff --git a/presto-main/src/test/java/com/facebook/presto/type/TestTimestampBase.java b/presto-main/src/test/java/com/facebook/presto/type/TestTimestampBase.java index 6c26c11baa94f..c1da9e5d68465 100644 --- a/presto-main/src/test/java/com/facebook/presto/type/TestTimestampBase.java +++ b/presto-main/src/test/java/com/facebook/presto/type/TestTimestampBase.java @@ -24,6 +24,8 @@ import org.joda.time.DateTimeZone; import org.testng.annotations.Test; +import java.time.ZoneId; +import java.util.TimeZone; import java.util.concurrent.TimeUnit; import static com.facebook.presto.common.function.OperatorType.INDETERMINATE; @@ -203,7 +205,7 @@ public void testCastToTimestampWithTimeZone() { assertFunction("cast(TIMESTAMP '2001-1-22 03:04:05.321' as timestamp with time zone)", TIMESTAMP_WITH_TIME_ZONE, - new SqlTimestampWithTimeZone(new DateTime(2001, 1, 22, 3, 4, 5, 321, DATE_TIME_ZONE).getMillis(), DATE_TIME_ZONE.toTimeZone())); + new SqlTimestampWithTimeZone(new DateTime(2001, 1, 22, 3, 4, 5, 321, DATE_TIME_ZONE).getMillis(), TimeZone.getTimeZone(ZoneId.of(DATE_TIME_ZONE.getID())))); functionAssertions.assertFunctionString("cast(TIMESTAMP '2001-1-22 03:04:05.321' as timestamp with time zone)", TIMESTAMP_WITH_TIME_ZONE, "2001-01-22 03:04:05.321 " + DATE_TIME_ZONE.getID()); diff --git a/presto-main/src/test/java/com/facebook/presto/util/TestTimeZoneUtils.java b/presto-main/src/test/java/com/facebook/presto/util/TestTimeZoneUtils.java index 897a01cb604a6..f8711fa3ce786 100644 --- a/presto-main/src/test/java/com/facebook/presto/util/TestTimeZoneUtils.java +++ b/presto-main/src/test/java/com/facebook/presto/util/TestTimeZoneUtils.java @@ -22,16 +22,20 @@ import java.util.TreeSet; import static com.facebook.presto.common.type.DateTimeEncoding.packDateTimeWithZone; +import static com.facebook.presto.common.type.DateTimeEncoding.unpackMillisUtc; +import static com.facebook.presto.common.type.DateTimeEncoding.unpackZoneKey; import static com.facebook.presto.common.type.TimeZoneKey.isUtcZoneId; import static com.facebook.presto.util.DateTimeZoneIndex.getDateTimeZone; import static com.facebook.presto.util.DateTimeZoneIndex.packDateTimeWithZone; import static com.facebook.presto.util.DateTimeZoneIndex.unpackDateTimeZone; +import static java.lang.String.format; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.fail; public class TestTimeZoneUtils { @Test - public void test() + public void testNamedZones() { TimeZoneKey.getTimeZoneKey("GMT-13:00"); @@ -50,13 +54,23 @@ public void test() continue; } + if (zoneId.equals("America/Godthab")) { + // TODO: Remove once minimum Java version is increased to 8u261 and 11.0.8 + // https://www.oracle.com/java/technologies/tzdata-versions.html + continue; + } + DateTimeZone dateTimeZone = DateTimeZone.forID(zoneId); DateTimeZone indexedZone = getDateTimeZone(TimeZoneKey.getTimeZoneKey(zoneId)); assertDateTimeZoneEquals(zoneId, indexedZone); assertTimeZone(zoneId, dateTimeZone); } + } + @Test + public void testOffsets() + { for (int offsetHours = -13; offsetHours < 14; offsetHours++) { for (int offsetMinutes = 0; offsetMinutes < 60; offsetMinutes++) { DateTimeZone dateTimeZone = DateTimeZone.forOffsetHoursMinutes(offsetHours, offsetMinutes); @@ -67,9 +81,21 @@ public void test() public static void assertTimeZone(String zoneId, DateTimeZone dateTimeZone) { - long dateTimeWithTimeZone = packDateTimeWithZone(new DateTime(42, dateTimeZone)); - assertEquals(packDateTimeWithZone((long) 42, dateTimeZone.toTimeZone().getID()), dateTimeWithTimeZone); - DateTimeZone unpackedZone = unpackDateTimeZone(dateTimeWithTimeZone); + long packWithDateTime = packDateTimeWithZone(new DateTime(42, dateTimeZone)); + long packWithZoneId = packDateTimeWithZone(42L, ZoneId.of(dateTimeZone.getID()).getId()); + if (packWithDateTime != packWithZoneId) { + fail(format( + "packWithDateTime and packWithZoneId differ for zone [%s] / [%s]: %s [%s %s] and %s [%s %s]", + zoneId, + dateTimeZone, + packWithDateTime, + unpackMillisUtc(packWithDateTime), + unpackZoneKey(packWithDateTime), + packWithZoneId, + unpackMillisUtc(packWithZoneId), + unpackZoneKey(packWithZoneId))); + } + DateTimeZone unpackedZone = unpackDateTimeZone(packWithDateTime); assertDateTimeZoneEquals(zoneId, unpackedZone); } diff --git a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManagerConfig.java b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManagerConfig.java index 6e5424b28a0c7..540d4251b48d8 100644 --- a/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManagerConfig.java +++ b/presto-raptor/src/main/java/com/facebook/presto/raptor/storage/StorageManagerConfig.java @@ -31,6 +31,7 @@ import javax.validation.constraints.NotNull; import java.net.URI; +import java.time.ZoneId; import java.util.TimeZone; import java.util.concurrent.TimeUnit; @@ -438,7 +439,7 @@ public StorageManagerConfig setOneSplitPerBucketThreshold(int oneSplitPerBucketT public DateTimeZone getShardDayBoundaryTimeZone() { - return DateTimeZone.forTimeZone(TimeZone.getTimeZone(shardDayBoundaryTimeZone)); + return DateTimeZone.forTimeZone(TimeZone.getTimeZone(ZoneId.of(shardDayBoundaryTimeZone, ZoneId.SHORT_IDS))); } @Config("storage.shard-day-boundary-time-zone") diff --git a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestTemporalFunction.java b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestTemporalFunction.java index 76ebbb1bfc67d..d7eb3e2c19aa0 100644 --- a/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestTemporalFunction.java +++ b/presto-raptor/src/test/java/com/facebook/presto/raptor/storage/organization/TestTemporalFunction.java @@ -20,6 +20,7 @@ import org.testng.annotations.Test; import java.time.Duration; +import java.time.ZoneId; import java.util.TimeZone; import static com.facebook.presto.common.type.DateType.DATE; @@ -29,7 +30,7 @@ public class TestTemporalFunction { - private static final DateTimeZone PST = DateTimeZone.forTimeZone(TimeZone.getTimeZone("PST")); + private static final DateTimeZone PST = DateTimeZone.forTimeZone(TimeZone.getTimeZone(ZoneId.of("PST", ZoneId.SHORT_IDS))); private static final DateTime UTC_TIME = new DateTime(1970, 1, 2, 0, 0, 0, UTC); private static final DateTime PST_TIME = new DateTime(1970, 1, 2, 0, 0, 0, PST); diff --git a/src/modernizer/violations.xml b/src/modernizer/violations.xml index 5cc8cd4c9b7b3..bca7ebcf3900a 100644 --- a/src/modernizer/violations.xml +++ b/src/modernizer/violations.xml @@ -29,4 +29,17 @@ 1.8 Use com.facebook.presto.testing.assertions.Assert.assertEquals due to TestNG #543 + + + java/util/TimeZone.getTimeZone:(Ljava/lang/String;)Ljava/util/TimeZone; + 1.8 + Avoid TimeZone.getTimeZone as it returns GMT for a zone not supported by the JVM. Use TimeZone.getTimeZone(ZoneId.of(..)) instead, or TimeZone.getTimeZone(..., false). + + + + org/joda/time/DateTimeZone.toTimeZone:()Ljava/util/TimeZone; + 1.8 + Avoid DateTimeZone.toTimeZone as it returns GMT for a zone not supported by the JVM. Use TimeZone.getTimeZone(ZoneId.of(dtz.getId())) instead. + +