ARROW-1281: [C++/Python] Add Docker setup for testing HDFS IO in C++ and Python

python/testing/README.md

@@ -0,0 +1,26 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# Testing tools for odds and ends
+
+## Testing HDFS file interface
+
+```shell
+./test_hdfs.sh
+```

python/testing/functions.sh

@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+use_gcc() {
+  export CC=gcc-4.9
+  export CXX=g++-4.9
+}
+
+use_clang() {
+  export CC=clang-4.0
+  export CXX=clang++-4.0
+}
+
+bootstrap_python_env() {
+  PYTHON_VERSION=$1
+  CONDA_ENV_DIR=$BUILD_DIR/pyarrow-test-$PYTHON_VERSION
+
+  conda create -y -q -p $CONDA_ENV_DIR python=$PYTHON_VERSION cmake curl
+  source activate $CONDA_ENV_DIR
+
+  python --version
+  which python
+
+  # faster builds, please
+  conda install -y -q nomkl pip numpy pandas cython
+}
+
+build_pyarrow() {
+  # Other stuff pip install
+  pushd $ARROW_PYTHON_DIR
+  pip install -r requirements.txt
+  python setup.py build_ext --with-parquet --with-plasma \
+         install --single-version-externally-managed --record=record.text
+  popd
+
+  python -c "import pyarrow.parquet"
+  python -c "import pyarrow.plasma"
+
+  export PYARROW_PATH=$CONDA_PREFIX/lib/python$PYTHON_VERSION/site-packages/pyarrow
+}
+
+build_arrow() {
+  mkdir -p $ARROW_CPP_BUILD_DIR
+  pushd $ARROW_CPP_BUILD_DIR
+
+  cmake -GNinja \
+        -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
+        -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
+        -DARROW_NO_DEPRECATED_API=ON \
+        -DARROW_PYTHON=ON \
+        -DARROW_PLASMA=ON \
+        -DARROW_BOOST_USE_SHARED=off \
+        $ARROW_CPP_DIR
+
+  ninja
+  ninja install
+  popd
+}
+
+build_parquet() {
+  PARQUET_DIR=$BUILD_DIR/parquet
+  mkdir -p $PARQUET_DIR
+
+  git clone https://github.com/apache/parquet-cpp.git $PARQUET_DIR
+
+  pushd $PARQUET_DIR
+  mkdir build-dir
+  cd build-dir
+
+  cmake \
+      -GNinja \
+      -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
+      -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
+      -DPARQUET_BOOST_USE_SHARED=off \
+      -DPARQUET_BUILD_BENCHMARKS=off \
+      -DPARQUET_BUILD_EXECUTABLES=off \
+      -DPARQUET_BUILD_TESTS=off \
+      ..
+
+  ninja
+  ninja install
+
+  popd
+}

python/testing/hdfs/Dockerfile

@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# TODO Replace this with a complete clean image build
+FROM cpcloud86/impala:metastore
+
+USER root
+
+RUN apt-add-repository -y ppa:ubuntu-toolchain-r/test && \
+    apt-get update && \
+    apt-get install -y \
+            gcc-4.9 \
+            g++-4.9 \
+            build-essential \
+            autotools-dev \
+            autoconf \
+            gtk-doc-tools \
+            autoconf-archive \
+            libgirepository1.0-dev \
+            libtool \
+            libjemalloc-dev \
+            ccache \
+            valgrind \
+            gdb
+
+RUN wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key|sudo apt-key add - && \
+    apt-add-repository -y \
+     "deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty-4.0 main" && \
+    apt-get update && \
+    apt-get install -y clang-4.0 clang-format-4.0 clang-tidy-4.0
+
+USER ubuntu
+
+RUN wget -O /tmp/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+    bash /tmp/miniconda.sh -b -p /home/ubuntu/miniconda && \
+    rm /tmp/miniconda.sh