ORC-418. Fix broken docker builds.

c++/include/orc/Exceptions.hh

@@ -30,7 +30,7 @@ namespace orc {
   public:
     explicit NotImplementedYet(const std::string& what_arg);
     explicit NotImplementedYet(const char* what_arg);
-    virtual ~NotImplementedYet() noexcept;
+    virtual ~NotImplementedYet() ORC_NOEXCEPT;
     NotImplementedYet(const NotImplementedYet&);
   private:
     NotImplementedYet& operator=(const NotImplementedYet&);
@@ -40,7 +40,7 @@ namespace orc {
   public:
     explicit ParseError(const std::string& what_arg);
     explicit ParseError(const char* what_arg);
-    virtual ~ParseError() noexcept;
+    virtual ~ParseError() ORC_NOEXCEPT;
     ParseError(const ParseError&);
   private:
     ParseError& operator=(const ParseError&);
@@ -50,7 +50,7 @@ namespace orc {
   public:
     explicit InvalidArgument(const std::string& what_arg);
     explicit InvalidArgument(const char* what_arg);
-    virtual ~InvalidArgument() noexcept;
+    virtual ~InvalidArgument() ORC_NOEXCEPT;
     InvalidArgument(const InvalidArgument&);
   private:
     InvalidArgument& operator=(const InvalidArgument&);

c++/src/Adaptor.hh.in

@@ -24,6 +24,8 @@
 #cmakedefine HAS_STRPTIME
 #cmakedefine HAS_STOLL
 #cmakedefine HAS_DIAGNOSTIC_PUSH
+#cmakedefine HAS_DOUBLE_TO_STRING
+#cmakedefine HAS_INT64_TO_STRING
 #cmakedefine HAS_PRE_1970
 #cmakedefine HAS_POST_2038
 #cmakedefine HAS_STD_ISNAN

c++/src/CMakeLists.txt

@@ -98,6 +98,25 @@ CHECK_CXX_SOURCE_COMPILES("
   NEEDS_REDUNDANT_MOVE
 )
 
+CHECK_CXX_SOURCE_COMPILES("
+    #include<string>
+    int main(int, char *[]) {
+      double d = 5;
+      std::to_string(d);
+    }"
+  HAS_DOUBLE_TO_STRING
+)
+
+CHECK_CXX_SOURCE_COMPILES("
+    #include<cstdint>
+    #include<string>
+    int main(int, char *[]) {
+      int64_t d = 5;
+      std::to_string(d);
+    }"
+  HAS_INT64_TO_STRING
+)
+
 INCLUDE(CheckCXXSourceRuns)
 
 CHECK_CXX_SOURCE_RUNS("
@@ -204,4 +223,4 @@ target_link_libraries (orc
   ${LIBHDFSPP_LIBRARIES}
   )
 
-install(TARGETS orc DESTINATION lib)
+install(TARGETS orc DESTINATION lib)

c++/src/Exceptions.cc

@@ -35,7 +35,7 @@ namespace orc {
     // PASS
   }
 
-  NotImplementedYet::~NotImplementedYet() noexcept {
+  NotImplementedYet::~NotImplementedYet() ORC_NOEXCEPT {
     // PASS
   }
 
@@ -53,7 +53,7 @@ namespace orc {
     // PASS
   }
 
-  ParseError::~ParseError() noexcept {
+  ParseError::~ParseError() ORC_NOEXCEPT {
     // PASS
   }
 
@@ -72,7 +72,7 @@ namespace orc {
     // PASS
   }
 
-  InvalidArgument::~InvalidArgument() noexcept {
+  InvalidArgument::~InvalidArgument() ORC_NOEXCEPT {
     // PASS
   }
 }

c++/src/RLEv1.cc

@@ -25,14 +25,14 @@
 
 namespace orc {
 
-const int MINIMUM_REPEAT = 3;
-const int MAXIMUM_REPEAT = 127 + MINIMUM_REPEAT;
+const uint64_t MINIMUM_REPEAT = 3;
+const uint64_t MAXIMUM_REPEAT = 127 + MINIMUM_REPEAT;
 
 const int64_t BASE_128_MASK = 0x7f;
 
-const int MAX_DELTA = 127;
-const int MIN_DELTA = -128;
-const int MAX_LITERAL_SIZE = 128;
+const int64_t MAX_DELTA = 127;
+const int64_t MIN_DELTA = -128;
+const uint64_t MAX_LITERAL_SIZE = 128;
 
 RleEncoderV1::RleEncoderV1(
                           std::unique_ptr<BufferedOutputStream> outStream,

c++/src/RLEv1.hh

@@ -40,7 +40,7 @@ public:
 private:
     int64_t delta;
     bool repeat;
-    int tailRunLength;
+    uint64_t tailRunLength;
 
     void write(int64_t val) override;
     void writeValues();

c++/src/RleEncoderV2.cc

@@ -26,6 +26,26 @@
 
 namespace orc {
 
+#ifdef HAS_DOUBLE_TO_STRING
+  std::string to_string(double val) {
+    return std::to_string(val);
+  }
+#else
+  std::string to_string(double val) {
+    return std::to_string(static_cast<long double>(val));
+  }
+#endif
+
+#ifdef HAS_INT64_TO_STRING
+  std::string to_string(int64_t val) {
+    return std::to_string(val);
+  }
+#else
+  std::string to_string(int64_t val) {
+    return std::to_string(static_cast<long long int>(val));
+  }
+#endif
+
 /**
  * Compute the bits required to represent pth percentile value
  * @param data - array
@@ -34,7 +54,7 @@ namespace orc {
  */
 uint32_t RleEncoderV2::percentileBits(int64_t* data, size_t offset, size_t length, double p, bool reuseHist) {
     if ((p > 1.0) || (p <= 0.0)) {
-        throw InvalidArgument("Invalid p value: " + std::to_string(p));
+        throw InvalidArgument("Invalid p value: " + to_string(p));
     }
 
     if (!reuseHist) {
@@ -342,11 +362,13 @@ void RleEncoderV2::determineEncoding(EncodingOption& option) {
     // fixed values run >10 which cannot be encoded with SHORT_REPEAT
     if (option.min == max) {
         if (!option.isFixedDelta) {
-            throw InvalidArgument(std::to_string(option.min) + "==" + std::to_string(max) + ", isFixedDelta cannot be false");
+            throw InvalidArgument(to_string(option.min) + "==" +
+              to_string(max) + ", isFixedDelta cannot be false");
         }
 
         if(currDelta != 0) {
-            throw InvalidArgument(std::to_string(option.min) + "==" + std::to_string(max) + ", currDelta should be zero");
+            throw InvalidArgument(to_string(option.min) + "==" +
+            to_string(max) + ", currDelta should be zero");
         }
         option.fixedDelta = 0;
         option.encoding = DELTA;

c++/test/TestRleEncoder.cc

@@ -40,7 +40,7 @@ namespace orc {
     virtual void SetUp();
 
   protected:
-    bool alignBitpacking = true;
+    bool alignBitpacking;
     std::unique_ptr<RleEncoder> getEncoder(RleVersion version,
                                            MemoryOutputStream& memStream,
                                            bool isSigned);

c++/test/TestWriter.cc

@@ -78,7 +78,9 @@ namespace orc {
     virtual void SetUp();
 
     protected:
-      FileVersion fileVersion = FileVersion::v_0_11();
+      FileVersion fileVersion;
+    public:
+      WriterTest(): fileVersion(FileVersion::v_0_11()) {}
   };
 
   void WriterTest::SetUp() {

cmake_modules/ThirdpartyToolchain.cmake

@@ -135,6 +135,7 @@ else ()
   set(LZ4_INCLUDE_DIR "${LZ4_PREFIX}/include")
   set(LZ4_STATIC_LIB "${LZ4_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}lz4${CMAKE_STATIC_LIBRARY_SUFFIX}")
   set(LZ4_CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${LZ4_PREFIX}
+                     -DCMAKE_INSTALL_LIBDIR=lib
                      -DBUILD_SHARED_LIBS=OFF)
 
   if (CMAKE_VERSION VERSION_GREATER "3.7")
@@ -237,6 +238,7 @@ else ()
   set(PROTOBUF_PREFIX "${THIRDPARTY_DIR}/protobuf_ep-install")
   set(PROTOBUF_INCLUDE_DIR "${PROTOBUF_PREFIX}/include")
   set(PROTOBUF_CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${PROTOBUF_PREFIX}
+                          -DCMAKE_INSTALL_LIBDIR=lib
                           -DBUILD_SHARED_LIBS=OFF
                           -Dprotobuf_BUILD_TESTS=OFF)
   if (MSVC)

docker/.gitignore

@@ -0,0 +1 @@
+logs

docker/README

@@ -9,3 +9,7 @@ To run the docker scripts:
 1. cd $os
 2. docker build -t orc-$os .
 3. docker run orc-$os
+
+To clean up docker:
+1. docker container prune
+2. docker image prune

docker/centos6/Dockerfile

@@ -29,8 +29,9 @@ RUN yum install -y \
   gcc-c++ \
   gettext-devel \
   git \
-  java-1.7.0-openjdk \
-  java-1.7.0-openjdk-devel \
+  java-1.8.0-openjdk \
+  java-1.8.0-openjdk-devel \
+  libtool \
   make \
   openssl-devel \
   tar \
@@ -42,6 +43,22 @@ WORKDIR /root
 RUN wget "https://www.apache.org/dyn/closer.lua?action=download&filename=/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz" -O maven.tgz
 RUN tar xzf maven.tgz
 RUN ln -s /root/apache-maven-3.3.9/bin/mvn /usr/bin/mvn
+# install a local build of protobuf
+RUN wget "https://github.com/protocolbuffers/protobuf/archive/v2.5.0.tar.gz" \
+  -O protobuf.tgz
+RUN tar xzf protobuf.tgz
+RUN cd protobuf-2.5.0 && \
+    autoreconf -f -i -Wall,no-obsolete && \
+    ./configure && \
+    make install
+# install a local build of snappy
+RUN wget "https://github.com/google/snappy/archive/1.1.3.tar.gz" \
+  -O snappy.tgz
+RUN tar xzf snappy.tgz
+RUN cd snappy-1.1.3 && \
+    ./autogen.sh && \
+    ./configure && \
+    make install
 
 CMD git clone https://github.com/apache/orc.git -b master && \
   mkdir orc/build && \

docker/centos7/Dockerfile

@@ -30,16 +30,27 @@ RUN yum install -y \
   gcc-c++ \
   gettext-devel \
   git \
-  java-1.7.0-openjdk \
-  java-1.7.0-openjdk-devel \
+  java-1.8.0-openjdk \
+  java-1.8.0-openjdk-devel \
+  libtool \
   make \
   maven \
   openssl-devel \
+  tar \
+  wget \
   which \
   zlib-devel
 
 ENV TZ=America/Los_Angeles
 WORKDIR /root
+# install a local build of snappy
+RUN wget "https://github.com/google/snappy/archive/1.1.3.tar.gz" \
+  -O snappy.tgz
+RUN tar xzf snappy.tgz
+RUN cd snappy-1.1.3 && \
+    ./autogen.sh && \
+    ./configure && \
+    make install
 
 CMD git clone https://github.com/apache/orc.git -b master && \
   mkdir orc/build && \

docker/debian8/Dockerfile

@@ -20,19 +20,36 @@
 FROM debian:8
 MAINTAINER Owen O'Malley <[email protected]>
 
+RUN echo "deb http://ftp.debian.org/debian jessie-backports main" \
+  >> /etc/apt/sources.list
+
 RUN apt-get update
 RUN apt-get install -y \
+  autoconf \
   cmake \
   gcc \
   g++ \
   git \
   libsasl2-dev \
   libssl-dev \
+  libtool \
   make \
   maven \
-  openjdk-7-jdk
+  pkg-config \
+  wget
+RUN apt-get -t jessie-backports install -y openjdk-8-jdk
+RUN update-alternatives --set java \
+  /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
 
 WORKDIR /root
+# install a local build of snappy
+RUN wget "https://github.com/google/snappy/archive/1.1.3.tar.gz" \
+  -O snappy.tgz
+RUN tar xzf snappy.tgz
+RUN cd snappy-1.1.3 && \
+    ./autogen.sh && \
+    ./configure && \
+    make install
 
 CMD git clone https://github.com/apache/orc.git -b master && \
   mkdir orc/build && \

docker/os-list.txt

@@ -0,0 +1,6 @@
+centos6
+centos7
+debian8
+ubuntu12
+ubuntu14
+ubuntu16

docker/debian7/Dockerfile → docker/reinit.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -14,26 +15,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# ORC compile for Debian 7
-#
-
-FROM debian:7
-MAINTAINER Owen O'Malley <[email protected]>
-
-RUN apt-get update
-RUN apt-get install -y \
-  cmake \
-  gcc \
-  g++ \
-  git \
-  make \
-  maven \
-  openjdk-7-jdk
-
-WORKDIR /root
-
-CMD git clone https://github.com/apache/orc.git -b master && \
-  mkdir orc/build && \
-  cd orc/build && \
-  cmake .. && \ 
-  make package test-out
+start=`date`
+for os in `cat os-list.txt`; do
+  echo "Re-initialize $os"
+  ( cd $os && docker build --no-cache -t "orc-$os" . )
+done
+echo "Start: $start"
+echo "End:" `date`