From d9cead53fefbc0c60962bcd52e3602bcf143d1b9 Mon Sep 17 00:00:00 2001 From: Prasanna Rajaperumal Date: Tue, 20 Dec 2016 15:38:50 -0800 Subject: [PATCH 1/2] Added .travis.yml file --- .travis.yml | 1 + travis.yml | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 .travis.yml delete mode 100644 travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000000..f5c99a7f6655a --- /dev/null +++ b/.travis.yml @@ -0,0 +1 @@ +language: java \ No newline at end of file diff --git a/travis.yml b/travis.yml deleted file mode 100644 index dff5f3a5d02a7..0000000000000 --- a/travis.yml +++ /dev/null @@ -1 +0,0 @@ -language: java From 5d44ae3dbd6418c4d1dece9e74d19891da1faf1f Mon Sep 17 00:00:00 2001 From: Prasanna Rajaperumal Date: Tue, 20 Dec 2016 16:20:38 -0800 Subject: [PATCH 2/2] Make hoodie run on travis-ci Test logs > 4MB, which is a limit for travis-ci. Reducing the logs by setting appropriate log levels for tests Add sudo: required on travis.yml to get more memory for running the tests. (https://github.com/travis-ci/travis-ci/issues/5926) Fixed requirement that fsclient.lastDataFileForDataset always returns files in order --- .travis.yml | 3 ++- .../common/HoodieTestDataGenerator.java | 8 ------ ...j.properties => log4j-surefire.properties} | 7 ++--- .../test/resources/log4j-surefire.properties | 25 ++++++++++++++++++ .../test/resources/log4j-surefire.properties | 25 ++++++++++++++++++ .../hoodie/hive/client/HoodieFSClient.java | 4 ++- .../uber/hoodie/hive/HDroneDatasetTest.java | 3 ++- .../test/resources/log4j-surefire.properties | 26 +++++++++++++++++++ pom.xml | 14 +++++++--- 9 files changed, 97 insertions(+), 18 deletions(-) rename hoodie-client/src/test/resources/{log4j.properties => log4j-surefire.properties} (83%) create mode 100644 hoodie-common/src/test/resources/log4j-surefire.properties create mode 100644 hoodie-hadoop-mr/src/test/resources/log4j-surefire.properties create mode 100644 hoodie-hive/src/test/resources/log4j-surefire.properties diff --git a/.travis.yml b/.travis.yml index f5c99a7f6655a..16a4f68db4a6a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1 +1,2 @@ -language: java \ No newline at end of file +language: java +sudo: required diff --git a/hoodie-client/src/test/java/com/uber/hoodie/common/HoodieTestDataGenerator.java b/hoodie-client/src/test/java/com/uber/hoodie/common/HoodieTestDataGenerator.java index 301e10e994131..00010f95afbac 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/common/HoodieTestDataGenerator.java +++ b/hoodie-client/src/test/java/com/uber/hoodie/common/HoodieTestDataGenerator.java @@ -45,10 +45,6 @@ * Test data uses a toy Uber trips, data model. */ public class HoodieTestDataGenerator { - - - private static Logger logger = LogManager.getLogger(HoodieTestDataGenerator.class); - static class KeyPartition { HoodieKey key; String partitionPath; @@ -97,8 +93,6 @@ public List generateInserts(String commitTime, int n) throws IOExc kp.key = key; kp.partitionPath = partitionPath; existingKeysList.add(kp); - - logger.info(" GENERATING INSERT FOR :" + key + "," + record.getPartitionPath()); } return inserts; } @@ -109,7 +103,6 @@ public List generateUpdates(String commitTime, List for (HoodieRecord baseRecord: baseRecords) { HoodieRecord record = new HoodieRecord(baseRecord.getKey(), generateRandomValue(baseRecord.getKey(), commitTime)); updates.add(record); - logger.info(" GENERATING UPDATE FOR :" + baseRecord.getKey()); } return updates; } @@ -123,7 +116,6 @@ public List generateUpdates(String commitTime, int n) throws IOExc KeyPartition kp = existingKeysList.get(rand.nextInt(existingKeysList.size() - 1)); HoodieRecord record = new HoodieRecord(kp.key, generateRandomValue(kp.key, commitTime)); updates.add(record); - logger.info(" GENERATING UPDATE FOR :" + kp.key); } return updates; } diff --git a/hoodie-client/src/test/resources/log4j.properties b/hoodie-client/src/test/resources/log4j-surefire.properties similarity index 83% rename from hoodie-client/src/test/resources/log4j.properties rename to hoodie-client/src/test/resources/log4j-surefire.properties index 5a8b643fdc4d7..eab225528fa4a 100644 --- a/hoodie-client/src/test/resources/log4j.properties +++ b/hoodie-client/src/test/resources/log4j-surefire.properties @@ -5,7 +5,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -13,9 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # +log4j.rootLogger=WARN, A1 +log4j.category.com.uber=INFO +log4j.category.org.apache.parquet.hadoop=WARN -# Set root logger level to DEBUG and its only appender to A1. -log4j.rootLogger=INFO, A1 # A1 is set to be a ConsoleAppender. log4j.appender.A1=org.apache.log4j.ConsoleAppender # A1 uses PatternLayout. diff --git a/hoodie-common/src/test/resources/log4j-surefire.properties b/hoodie-common/src/test/resources/log4j-surefire.properties new file mode 100644 index 0000000000000..1c03f27e67ebd --- /dev/null +++ b/hoodie-common/src/test/resources/log4j-surefire.properties @@ -0,0 +1,25 @@ +# +# Copyright (c) 2016 Uber Technologies, Inc. (hoodie-dev-group@uber.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +log4j.rootLogger=WARN, A1 +log4j.category.com.uber=INFO +log4j.category.org.apache.parquet.hadoop=WARN + +# A1 is set to be a ConsoleAppender. +log4j.appender.A1=org.apache.log4j.ConsoleAppender +# A1 uses PatternLayout. +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n diff --git a/hoodie-hadoop-mr/src/test/resources/log4j-surefire.properties b/hoodie-hadoop-mr/src/test/resources/log4j-surefire.properties new file mode 100644 index 0000000000000..1c03f27e67ebd --- /dev/null +++ b/hoodie-hadoop-mr/src/test/resources/log4j-surefire.properties @@ -0,0 +1,25 @@ +# +# Copyright (c) 2016 Uber Technologies, Inc. (hoodie-dev-group@uber.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +log4j.rootLogger=WARN, A1 +log4j.category.com.uber=INFO +log4j.category.org.apache.parquet.hadoop=WARN + +# A1 is set to be a ConsoleAppender. +log4j.appender.A1=org.apache.log4j.ConsoleAppender +# A1 uses PatternLayout. +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n diff --git a/hoodie-hive/src/main/java/com/uber/hoodie/hive/client/HoodieFSClient.java b/hoodie-hive/src/main/java/com/uber/hoodie/hive/client/HoodieFSClient.java index 0436d5285b315..9fd95c004bb88 100644 --- a/hoodie-hive/src/main/java/com/uber/hoodie/hive/client/HoodieFSClient.java +++ b/hoodie-hive/src/main/java/com/uber/hoodie/hive/client/HoodieFSClient.java @@ -106,7 +106,9 @@ public Path lastDataFileForDataset(HoodieDatasetReference metadata, Path path = files.next().getPath(); if (path.getName().endsWith(PARQUET_EXTENSION) || path.getName() .endsWith(PARQUET_EXTENSION_ZIPPED)) { - returnPath = path; + if(returnPath == null || path.toString().compareTo(returnPath.toString()) > 0) { + returnPath = path; + } } } if (returnPath != null) { diff --git a/hoodie-hive/src/test/java/com/uber/hoodie/hive/HDroneDatasetTest.java b/hoodie-hive/src/test/java/com/uber/hoodie/hive/HDroneDatasetTest.java index cdcb528d1878e..2a0c11c233a21 100644 --- a/hoodie-hive/src/test/java/com/uber/hoodie/hive/HDroneDatasetTest.java +++ b/hoodie-hive/src/test/java/com/uber/hoodie/hive/HDroneDatasetTest.java @@ -92,7 +92,8 @@ public void testDatasetEvolution() throws IOException, InitializationError { assertEquals("Table schema should be evolved schema", expectedSchema, newDatasetSchema); assertEquals("Table schema should have 6 fields", 6, hiveClient.getTableSchema(metadata).size()); - assertEquals("", "BIGINT", hiveClient.getTableSchema(metadata).get("region_key")); + assertEquals("Valid Evolution should be reflected", "BIGINT", + hiveClient.getTableSchema(metadata).get("region_key")); } } diff --git a/hoodie-hive/src/test/resources/log4j-surefire.properties b/hoodie-hive/src/test/resources/log4j-surefire.properties new file mode 100644 index 0000000000000..cc6a57052b55e --- /dev/null +++ b/hoodie-hive/src/test/resources/log4j-surefire.properties @@ -0,0 +1,26 @@ +# +# Copyright (c) 2016 Uber Technologies, Inc. (hoodie-dev-group@uber.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +log4j.rootLogger=WARN, A1 +log4j.category.com.uber=INFO +log4j.category.org.apache.parquet.hadoop=WARN +log4j.category.parquet.hadoop=WARN + +# A1 is set to be a ConsoleAppender. +log4j.appender.A1=org.apache.log4j.ConsoleAppender +# A1 uses PatternLayout. +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n diff --git a/pom.xml b/pom.xml index 8553fb378c635..2c2384785ae5b 100644 --- a/pom.xml +++ b/pom.xml @@ -130,6 +130,16 @@ maven-release-plugin 2.5.3 + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + + file:${project.build.testOutputDirectory}/log4j-surefire.properties + + + @@ -141,10 +151,6 @@ maven-jar-plugin ${maven-jar-plugin.version} - - maven-surefire-plugin - ${maven-surefire-plugin.version} - org.codehaus.mojo cobertura-maven-plugin