diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index cd12be11488..eaccf25403e 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -23,6 +23,7 @@ on: - '.github/workflows/cpp.yml' - 'ci/docker/**' - 'ci/scripts/cpp_*' + - 'ci/scripts/install_azurite.sh' - 'ci/scripts/install_gcs_testbench.sh' - 'ci/scripts/install_minio.sh' - 'ci/scripts/msys2_*' @@ -34,6 +35,7 @@ on: - '.github/workflows/cpp.yml' - 'ci/docker/**' - 'ci/scripts/cpp_*' + - 'ci/scripts/install_azurite.sh' - 'ci/scripts/install_gcs_testbench.sh' - 'ci/scripts/install_minio.sh' - 'ci/scripts/msys2_*' @@ -201,6 +203,8 @@ jobs: ci/scripts/install_minio.sh latest /usr/local - name: Install Google Cloud Storage Testbench run: ci/scripts/install_gcs_testbench.sh default + - name: Install Azurite Storage Emulator + run: ci/scripts/install_azurite.sh - name: Setup ccache run: | ci/scripts/ccache_setup.sh diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index f94494177e8..125f1f48d48 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -103,6 +103,7 @@ RUN apt-get update -y -q && \ make \ ninja-build \ nlohmann-json3-dev \ + npm \ pkg-config \ protobuf-compiler \ python3-dev \ @@ -123,6 +124,9 @@ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_gcs_testbench.sh default +COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_azurite.sh + COPY ci/scripts/install_ceph.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_ceph.sh diff --git a/ci/docker/ubuntu-22.04-cpp.dockerfile b/ci/docker/ubuntu-22.04-cpp.dockerfile index e773c6f1ee6..0840b3fa5c6 100644 --- a/ci/docker/ubuntu-22.04-cpp.dockerfile +++ b/ci/docker/ubuntu-22.04-cpp.dockerfile @@ -102,6 +102,7 @@ RUN apt-get update -y -q && \ make \ ninja-build \ nlohmann-json3-dev \ + npm \ pkg-config \ protobuf-compiler \ protobuf-compiler-grpc \ @@ -153,6 +154,9 @@ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_gcs_testbench.sh default +COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_azurite.sh + COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin diff --git a/ci/scripts/install_azurite.sh b/ci/scripts/install_azurite.sh new file mode 100755 index 00000000000..2e7008360fd --- /dev/null +++ b/ci/scripts/install_azurite.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +case "$(uname)" in + Darwin) + npm install -g azurite + which azurite + ;; + MINGW*) + choco install nodejs.install + npm install -g azurite + ;; + Linux) + npm install -g azurite + which azurite + ;; +esac +echo "node version = $(node --version)" +echo "azurite version = $(azurite --version)" \ No newline at end of file diff --git a/cpp/Brewfile b/cpp/Brewfile index 580e8d3f115..58015d2121b 100644 --- a/cpp/Brewfile +++ b/cpp/Brewfile @@ -30,6 +30,7 @@ brew "grpc" brew "llvm@14" brew "lz4" brew "ninja" +brew "node" brew "openssl@3" brew "protobuf" brew "python" diff --git a/cpp/src/arrow/filesystem/azurefs_test.cc b/cpp/src/arrow/filesystem/azurefs_test.cc index 0f03e88393a..e940c5bd1bc 100644 --- a/cpp/src/arrow/filesystem/azurefs_test.cc +++ b/cpp/src/arrow/filesystem/azurefs_test.cc @@ -15,7 +15,26 @@ // specific language governing permissions and limitations // under the License. +#include // Missing include in boost/process + +// This boost/asio/io_context.hpp include is needless for no MinGW +// build. +// +// This is for including boost/asio/detail/socket_types.hpp before any +// "#include ". boost/asio/detail/socket_types.hpp doesn't +// work if windows.h is already included. boost/process.h -> +// boost/process/args.hpp -> boost/process/detail/basic_cmd.hpp +// includes windows.h. boost/process/args.hpp is included before +// boost/process/async.h that includes +// boost/asio/detail/socket_types.hpp implicitly is included. +#include +// We need BOOST_USE_WINDOWS_H definition with MinGW when we use +// boost/process.hpp. See BOOST_USE_WINDOWS_H=1 in +// cpp/cmake_modules/ThirdpartyToolchain.cmake for details. +#include + #include "arrow/filesystem/azurefs.h" +#include "arrow/util/io_util.h" #include #include @@ -27,15 +46,76 @@ #include "arrow/testing/util.h" namespace arrow { +using internal::TemporaryDir; namespace fs { namespace { +namespace bp = boost::process; using ::testing::IsEmpty; using ::testing::Not; using ::testing::NotNull; -// Placeholder test for file structure +class AzuriteEnv : public ::testing::Environment { + public: + AzuriteEnv() { + account_name_ = "devstoreaccount1"; + account_key_ = + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/" + "KBHBeksoGMGw=="; + auto exe_path = bp::search_path("azurite"); + if (exe_path.empty()) { + auto error = std::string("Could not find Azurite emulator."); + status_ = Status::Invalid(error); + return; + } + auto temp_dir_ = *TemporaryDir::Make("azurefs-test-"); + server_process_ = bp::child(boost::this_process::environment(), exe_path, "--silent", + "--location", temp_dir_->path().ToString(), "--debug", + temp_dir_->path().ToString() + "/debug.log"); + if (!(server_process_.valid() && server_process_.running())) { + auto error = "Could not start Azurite emulator."; + server_process_.terminate(); + server_process_.wait(); + status_ = Status::Invalid(error); + return; + } + status_ = Status::OK(); + } + + ~AzuriteEnv() override { + server_process_.terminate(); + server_process_.wait(); + } + + const std::string& account_name() const { return account_name_; } + const std::string& account_key() const { return account_key_; } + const Status status() const { return status_; } + + private: + std::string account_name_; + std::string account_key_; + bp::child server_process_; + Status status_; + std::unique_ptr temp_dir_; +}; + +auto* azurite_env = ::testing::AddGlobalTestEnvironment(new AzuriteEnv); + +AzuriteEnv* GetAzuriteEnv() { + return ::arrow::internal::checked_cast(azurite_env); +} + +// Placeholder tests for file structure // TODO: GH-18014 Remove once a proper test is added +TEST(AzureFileSystem, InitialiseAzurite) { + const std::string& account_name = GetAzuriteEnv()->account_name(); + const std::string& account_key = GetAzuriteEnv()->account_key(); + EXPECT_EQ(account_name, "devstoreaccount1"); + EXPECT_EQ(account_key, + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/" + "K1SZFPTOtr/KBHBeksoGMGw=="); +} + TEST(AzureFileSystem, OptionsCompare) { AzureOptions options; EXPECT_TRUE(options.Equals(options));