diff --git a/.gitmodules b/.gitmodules index 653dbbf76..7547a83fa 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,8 +2,13 @@ path = librocksdb_sys/rocksdb url = https://github.com/tikv/rocksdb.git branch = 6.4.tikv + [submodule "titan"] path = librocksdb_sys/libtitan_sys/titan url = https://github.com/tikv/titan.git branch = master +[submodule "rocksdb-cloud"] + path = librocksdb_sys/librocksdb_cloud_sys/rocksdb-cloud + url = https://github.com/tikv/rocksdb + branch = 6.4.cloud diff --git a/.travis.yml b/.travis.yml index 1e2022cac..c44cc97fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,14 @@ dist: bionic language: rust -os: - - linux - rust: - nightly +cache: + cargo: false + directories: + - "/tmp/aws-sdk-cpp-1.8.14" + addons: apt: sources: @@ -19,6 +21,9 @@ jobs: - os: linux rust: nightly env: FEATURES="encryption,jemalloc,portable,sse" + - os: linux + rust: nightly + env: FEATURES="cloud" - os: linux arch: arm64 rust: nightly @@ -31,14 +36,26 @@ env: global: - RUST_BACKTRACE=1 -cache: false - before_script: + # Installing the AWS-SDK, so increase the maximum number of open file descriptors, + # since some tests use more FDs than the default limit. + - if [ "${FEATURES}" == "cloud" ]; then + echo "limit maxfiles 1024 unlimited" | sudo tee -a /etc/launchd.conf; + pushd /tmp; + wget https://github.com/aws/aws-sdk-cpp/archive/1.8.14.tar.gz -O /tmp/aws-sdk.tar.gz; + tar -xvf /tmp/aws-sdk.tar.gz > /dev/null; + popd; + pushd /tmp/aws-sdk-cpp-1.8.14; + cmake -DBUILD_ONLY='kinesis;core;s3;transfer' -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_TESTING=OFF .; + make -j4 all; + sudo make install; + popd; + fi - rustup component add rustfmt-preview - rustup component add clippy script: - # copile rocksdb may cost more than 10 minutes, see https://docs.travis-ci.com/user/common-build-problems/#build-times-out-because-no-output-was-received + # compiling rocksdb may cost more than 10 minutes, see https://docs.travis-ci.com/user/common-build-problems/#build-times-out-because-no-output-was-received - if [ "${TASK}" == "clang-format" ]; then bash ./travis-build/travis-format.sh; else diff --git a/Cargo.toml b/Cargo.toml index a4cc9f99f..8da0fc52f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,8 @@ portable = ["librocksdb_sys/portable"] sse = ["librocksdb_sys/sse"] static_libcpp = ["librocksdb_sys/static_libcpp"] valgrind = [] +# rocksdb-cloud integration +cloud = ["librocksdb_sys/cloud"] [dependencies] libc = "0.2.11" diff --git a/Makefile b/Makefile index fb6b8b916..adbd4b630 100644 --- a/Makefile +++ b/Makefile @@ -41,3 +41,14 @@ update_rocksdb: fi @git submodule sync @git submodule update --init --remote librocksdb_sys/rocksdb + +update_rocksdb_cloud: + @if [ -n "${ROCKSDB_CLOUD_REPO}" ]; then \ + git config --file=.gitmodules submodule.rocksdb-cloud.url https://github.com/${ROCKSDB_CLOUD_REPO}/rocksdb.git; \ + fi + @if [ -n "${ROCKSDB_CLOUD_BRANCH}" ]; then \ + git config --file=.gitmodules submodule.rocksdb-cloud.branch ${ROCKSDB_CLOUD_BRANCH}; \ + fi + @git submodule sync + @git submodule update --init --remote librocksdb_sys/librocksdb_cloud_sys/rocksdb-cloud + \ No newline at end of file diff --git a/librocksdb_sys/Cargo.toml b/librocksdb_sys/Cargo.toml index 1a1e8a098..c94fbc231 100644 --- a/librocksdb_sys/Cargo.toml +++ b/librocksdb_sys/Cargo.toml @@ -8,6 +8,7 @@ links = "rocksdb" [dependencies] bzip2-sys = "0.1.8+1.0.8" libc = "0.2.11" +librocksdb_cloud_sys = { path = "librocksdb_cloud_sys" } libtitan_sys = { path = "libtitan_sys" } libz-sys = { version = "1.0.25", features = ["static"] } openssl-sys = { version = "0.9.54", optional = true, features = ["vendored"] } @@ -25,6 +26,7 @@ jemalloc = ["tikv-jemalloc-sys"] portable = ["libtitan_sys/portable"] sse = ["libtitan_sys/sse"] static_libcpp = [] +cloud = [] [build-dependencies] cc = "1.0.3" diff --git a/librocksdb_sys/build.rs b/librocksdb_sys/build.rs index 56154302e..5008f0baf 100644 --- a/librocksdb_sys/build.rs +++ b/librocksdb_sys/build.rs @@ -128,6 +128,10 @@ fn link_cpp(build: &mut Build) { fn build_rocksdb() -> Build { let target = env::var("TARGET").expect("TARGET was not set"); let mut cfg = Config::new("rocksdb"); + // Conditionally compile with support for RocksDB-Cloud, setting USE_AWS + if cfg!(feature = "cloud") { + println!("cargo:rustc-link-lib=static=rocksdb_cloud"); + } if cfg!(feature = "encryption") { cfg.register_dep("OPENSSL").define("WITH_OPENSSL", "ON"); println!("cargo:rustc-link-lib=static=crypto"); @@ -195,6 +199,13 @@ fn build_rocksdb() -> Build { build.include(cur_dir.join("rocksdb")); build.include(cur_dir.join("libtitan_sys").join("titan").join("include")); build.include(cur_dir.join("libtitan_sys").join("titan")); + build.include( + cur_dir + .join("librocksdb_cloud_sys") + .join("rocksdb-cloud") + .join("include"), + ); + build.include(cur_dir.join("librocksdb_cloud_sys").join("rocksdb-cloud")); // Adding rocksdb specific compile macros. // TODO: should make sure crocksdb compile options is the same as rocksdb and titan. @@ -202,6 +213,9 @@ fn build_rocksdb() -> Build { if cfg!(feature = "encryption") { build.define("OPENSSL", None); } + if cfg!(feature = "cloud") { + build.define("USE_AWS", None).define("USE_CLOUD", None); + } println!("cargo:rustc-link-lib=static=rocksdb"); println!("cargo:rustc-link-lib=static=titan"); diff --git a/librocksdb_sys/crocksdb/c.cc b/librocksdb_sys/crocksdb/c.cc index dfde91a0d..fdc6e5530 100644 --- a/librocksdb_sys/crocksdb/c.cc +++ b/librocksdb_sys/crocksdb/c.cc @@ -15,6 +15,7 @@ #include "db/column_family.h" #include "rocksdb/cache.h" +#include "rocksdb/cloud/cloud_env_options.h" #include "rocksdb/compaction_filter.h" #include "rocksdb/comparator.h" #include "rocksdb/convenience.h" @@ -178,6 +179,9 @@ using rocksdb::titandb::TitanDBOptions; using rocksdb::titandb::TitanOptions; using rocksdb::titandb::TitanReadOptions; +using rocksdb::CloudEnv; +using rocksdb::CloudEnvOptions; + using rocksdb::MemoryAllocator; #ifdef OPENSSL @@ -5853,4 +5857,49 @@ void ctitandb_delete_files_in_ranges_cf( SaveError(errptr, static_cast(db->rep)->DeleteFilesInRanges( cf->rep, &ranges[0], num_ranges, include_end)); } + +/* RocksDB Cloud */ +#ifdef USE_CLOUD +struct crocksdb_cloud_envoptions_t { + CloudEnvOptions rep; +}; + +crocksdb_env_t* crocksdb_cloud_aws_env_create( + crocksdb_env_t* base_env, const char* src_cloud_bucket, + const char* src_cloud_object, const char* src_cloud_region, + const char* dest_cloud_bucket, const char* dest_cloud_object, + const char* dest_cloud_region, crocksdb_cloud_envoptions_t* cloud_options, + char** errptr) { + // Store a reference to a cloud env. A new cloud env object should be + // associated with every new cloud-db. + CloudEnv* cloud_env; + + CloudEnv* cenv; + if (SaveError(errptr, + CloudEnv::NewAwsEnv( + base_env->rep, src_cloud_bucket, src_cloud_object, + src_cloud_region, dest_cloud_bucket, dest_cloud_object, + dest_cloud_region, cloud_options->rep, nullptr, &cenv))) { + assert(cenv != nullptr); + return nullptr; + } + cloud_env = cenv; + + crocksdb_env_t* result = new crocksdb_env_t; + result->rep = static_cast(cloud_env); + result->block_cipher = nullptr; + result->encryption_provider = nullptr; + result->is_default = true; + return result; +} + +crocksdb_cloud_envoptions_t* crocksdb_cloud_envoptions_create() { + crocksdb_cloud_envoptions_t* opt = new crocksdb_cloud_envoptions_t; + return opt; +} + +void crocksdb_cloud_envoptions_destroy(crocksdb_cloud_envoptions_t* opt) { + delete opt; +} +#endif } // end extern "C" diff --git a/librocksdb_sys/crocksdb/crocksdb/c.h b/librocksdb_sys/crocksdb/crocksdb/c.h index 2bb6e5ae7..a0ede950b 100644 --- a/librocksdb_sys/crocksdb/crocksdb/c.h +++ b/librocksdb_sys/crocksdb/crocksdb/c.h @@ -70,6 +70,7 @@ extern "C" { /* Exported types */ +typedef struct crocksdb_cloud_envoptions_t crocksdb_cloud_envoptions_t; typedef struct crocksdb_t crocksdb_t; typedef struct crocksdb_status_ptr_t crocksdb_status_ptr_t; typedef struct crocksdb_backup_engine_t crocksdb_backup_engine_t; @@ -2348,6 +2349,21 @@ extern C_ROCKSDB_LIBRARY_API void ctitandb_delete_files_in_ranges_cf( const char* const* limit_keys, const size_t* limit_keys_lens, size_t num_ranges, unsigned char include_end, char** errptr); +/* RocksDB Cloud */ + +#ifdef USE_CLOUD +extern C_ROCKSDB_LIBRARY_API crocksdb_env_t* crocksdb_cloud_aws_env_create( + crocksdb_env_t* base_env, const char* src_cloud_bucket, + const char* src_cloud_object, const char* src_cloud_region, + const char* dest_cloud_bucket, const char* dest_cloud_object, + const char* dest_cloud_region, crocksdb_cloud_envoptions_t* cloud_options, + char** errptr); +extern C_ROCKSDB_LIBRARY_API crocksdb_cloud_envoptions_t* +crocksdb_cloud_envoptions_create(); +extern C_ROCKSDB_LIBRARY_API void crocksdb_cloud_envoptions_destroy( + crocksdb_cloud_envoptions_t* opt); +#endif + #ifdef __cplusplus } /* end extern "C" */ #endif diff --git a/librocksdb_sys/librocksdb_cloud_sys/CMakeLists.txt b/librocksdb_sys/librocksdb_cloud_sys/CMakeLists.txt new file mode 100644 index 000000000..29a88c8ae --- /dev/null +++ b/librocksdb_sys/librocksdb_cloud_sys/CMakeLists.txt @@ -0,0 +1,60 @@ +# Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0. +cmake_minimum_required(VERSION 3.0) +project(rocksdb_cloud) +enable_language(CXX) +enable_language(C) + +# Manage absence of rocksdb code dependence + +if (NOT ROCKSDB_GIT_REPO) + set(ROCKSDB_GIT_REPO "https://github.com/tikv/rocksdb.git") +endif() + +if (NOT ROCKSDB_GIT_BRANCH) + set(ROCKSDB_GIT_BRANCH "6.4.tikv") +endif() + +if (NOT DEFINED ROCKSDB_DIR) + if (GIT_FOUND) + if (WIN32) + execute_process(COMMAND $ENV{COMSPEC} /C ${GIT_EXECUTABLE} clone --branch=${ROCKSDB_GIT_BRANCH} ${ROCKSDB_GIT_REPO}) + else() + execute_process(COMMAND ${GIT_EXECUTABLE} clone --branch=${ROCKSDB_GIT_BRANCH} ${ROCKSDB_GIT_REPO}) + endif() + set(ROCKSDB_DIR "${CMAKE_BINARY_DIR}/rocksdb") + endif() +endif() + +if (NOT DEFINED ROCKSDB_DIR) + message(FATAL_ERROR "ROCKSDB_DIR is not defined.") +endif() + +# Append rocksdb modules +list(APPEND CMAKE_MODULE_PATH "${ROCKSDB_DIR}/cmake/modules/") +include("cmake_modules/rocksdb_flags.cmake") + +include_directories("${ROCKSDB_DIR}") +include_directories("${ROCKSDB_DIR}/include") +include_directories("rocksdb-cloud") +include_directories("rocksdb-cloud/include") + +file(GLOB SOURCES rocksdb-cloud/cloud/*.cc) + +add_library(rocksdb_cloud STATIC ${SOURCES} $) + +option(WITH_CLOUD_TESTS "Build with tests." ON) + +include_directories(SYSTEM ${ROCKSDB_DIR}/third-party/gtest-1.7.0/fused-src) + +add_library(cloud_env OBJECT "rocksdb-cloud/cloud/cloud_env.cc") + +include(GNUInstallDirs) +install(DIRECTORY include/rocksdb/cloud + COMPONENT devel + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" +) +install(TARGETS rocksdb_cloud + COMPONENT devel + ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" + INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" +) diff --git a/librocksdb_sys/librocksdb_cloud_sys/Cargo.toml b/librocksdb_sys/librocksdb_cloud_sys/Cargo.toml new file mode 100644 index 000000000..add285959 --- /dev/null +++ b/librocksdb_sys/librocksdb_cloud_sys/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "librocksdb_cloud_sys" +version = "0.1.0" +edition = "2018" +build = "build.rs" +links = "rocksdb-cloud" + +[features] +cloud = [] + +[build-dependencies] +cc = "1.0.3" +cmake = "0.1" diff --git a/librocksdb_sys/librocksdb_cloud_sys/build.rs b/librocksdb_sys/librocksdb_cloud_sys/build.rs new file mode 100644 index 000000000..19ca5d60f --- /dev/null +++ b/librocksdb_sys/librocksdb_cloud_sys/build.rs @@ -0,0 +1,18 @@ +// Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0. +extern crate cc; +extern crate cmake; + +use std::env; + +fn main() { + let cur_dir = env::current_dir().unwrap(); + let mut cfg = cmake::Config::new("."); + let dst = cfg + .define("ROCKSDB_DIR", cur_dir.join("..").join("rocksdb")) + .env("USE_AWS", "1") + .build_target("rocksdb_cloud") + .very_verbose(true) + .build(); + println!("cargo:rustc-link-search=native={}/build", dst.display()); + println!("cargo:rustc-link-lib=static=rocksdb_cloud"); +} diff --git a/librocksdb_sys/librocksdb_cloud_sys/cmake_modules/rocksdb_flags.cmake b/librocksdb_sys/librocksdb_cloud_sys/cmake_modules/rocksdb_flags.cmake new file mode 100644 index 000000000..d51cfe2ed --- /dev/null +++ b/librocksdb_sys/librocksdb_cloud_sys/cmake_modules/rocksdb_flags.cmake @@ -0,0 +1,325 @@ +# Generate compile and link flags +# Extracted from rocksdb/CMakeLists.txt + + +option(WITH_JEMALLOC "build with JeMalloc" OFF) +if (WITH_JEMALLOC) + find_package(JeMalloc REQUIRED) + add_definitions(-DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE) + include_directories(${JEMALLOC_INCLUDE_DIR}) +endif() + +option(WITH_SNAPPY "build with SNAPPY" OFF) +if (WITH_SNAPPY) + find_package(snappy REQUIRED) + add_definitions(-DSNAPPY) + include_directories(${SNAPPY_INCLUDE_DIR}) +endif() + +option(WITH_BZ2 "build with bzip2" OFF) +if (WITH_BZ2) + find_package(bzip2 REQUIRED) + add_definitions(-DBZIP2) + include_directories(${BZIP2_INCLUDE_DIR}) +endif() + +option(WITH_LZ4 "build with lz4" OFF) +if (WITH_LZ4) + find_package(lz4 REQUIRED) + add_definitions(-DLZ4) + include_directories(${LZ4_INCLUDE_DIR}) +endif() + +option(WITH_ZLIB "build with zlib" OFF) +if (WITH_ZLIB) + find_package(ZLIB REQUIRED) + add_definitions(-DZLIB) + include_directories(${ZLIB_INCLUDE_DIR}) +endif() + +option(WITH_ZSTD "build with zstd" OFF) +if (WITH_ZSTD) + find_package(zstd REQUIRED) + add_definitions(-DZSTD) + include_directories(${ZSTD_INCLUDE_DIR}) +endif() + +if(MSVC) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W4 /wd4127 /wd4800 /wd4996 /wd4351 /wd4100 /wd4204 /wd4324") +else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers -Wno-strict-aliasing") + if(MINGW) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format") + endif() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer") + include(CheckCXXCompilerFlag) + CHECK_CXX_COMPILER_FLAG("-momit-leaf-frame-pointer" HAVE_OMIT_LEAF_FRAME_POINTER) + if(HAVE_OMIT_LEAF_FRAME_POINTER) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -momit-leaf-frame-pointer") + endif() + endif() +endif() + +include(CheckCCompilerFlag) +if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le") + CHECK_C_COMPILER_FLAG("-maltivec" HAS_ALTIVEC) + if(HAS_ALTIVEC) + message(STATUS " HAS_ALTIVEC yes") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -maltivec") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -maltivec") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mcpu=power8") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=power8") + endif(HAS_ALTIVEC) +endif(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le") + +if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64") + CHECK_C_COMPILER_FLAG("-march=armv8-a+crc" HAS_ARMV8_CRC) + if(HAS_ARMV8_CRC) + message(STATUS " HAS_ARMV8_CRC yes") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=armv8-a+crc -Wno-unused-function") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=armv8-a+crc -Wno-unused-function") + endif(HAS_ARMV8_CRC) +endif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64") + +option(PORTABLE "build a portable binary" OFF) +option(FORCE_SSE42 "force building with SSE4.2, even when PORTABLE=ON" OFF) +if(PORTABLE) + # MSVC does not need a separate compiler flag to enable SSE4.2; if nmmintrin.h + # is available, it is available by default. + if(FORCE_SSE42 AND NOT MSVC) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2 -mpclmul") + endif() +else() + if(MSVC) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2") + else() + if(NOT HAVE_POWER8 AND NOT HAS_ARMV8_CRC) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native") + endif() + endif() +endif() + +include(CheckCXXSourceCompiles) +if(NOT MSVC) + set(CMAKE_REQUIRED_FLAGS "-msse4.2 -mpclmul") +endif() +CHECK_CXX_SOURCE_COMPILES(" +#include +#include +#include +int main() { + volatile uint32_t x = _mm_crc32_u32(0, 0); + const auto a = _mm_set_epi64x(0, 0); + const auto b = _mm_set_epi64x(0, 0); + const auto c = _mm_clmulepi64_si128(a, b, 0x00); + auto d = _mm_cvtsi128_si64(c); +} +" HAVE_SSE42) +unset(CMAKE_REQUIRED_FLAGS) +if(HAVE_SSE42) + add_definitions(-DHAVE_SSE42) + add_definitions(-DHAVE_PCLMUL) +elseif(FORCE_SSE42) + message(FATAL_ERROR "FORCE_SSE42=ON but unable to compile with SSE4.2 enabled") +endif() + +CHECK_CXX_SOURCE_COMPILES(" +#if defined(_MSC_VER) && !defined(__thread) +#define __thread __declspec(thread) +#endif +int main() { + static __thread int tls; +} +" HAVE_THREAD_LOCAL) +if(HAVE_THREAD_LOCAL) + add_definitions(-DROCKSDB_SUPPORT_THREAD_LOCAL) +endif() + +option(FAIL_ON_WARNINGS "Treat compile warnings as errors" ON) +if(FAIL_ON_WARNINGS) + if(MSVC) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /WX") + else() # assume GCC + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror") + endif() +endif() + +option(WITH_ASAN "build with ASAN" OFF) +if(WITH_ASAN) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address") + if(WITH_JEMALLOC) + message(FATAL "ASAN does not work well with JeMalloc") + endif() +endif() + +option(WITH_TSAN "build with TSAN" OFF) +if(WITH_TSAN) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread -pie") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread -fPIC") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=thread -fPIC") + if(WITH_JEMALLOC) + message(FATAL "TSAN does not work well with JeMalloc") + endif() +endif() + +option(WITH_UBSAN "build with UBSAN" OFF) +if(WITH_UBSAN) + add_definitions(-DROCKSDB_UBSAN_RUN) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=undefined") + if(WITH_JEMALLOC) + message(FATAL "UBSAN does not work well with JeMalloc") + endif() +endif() + +# Stall notifications eat some performance from inserts +option(DISABLE_STALL_NOTIF "Build with stall notifications" OFF) +if(DISABLE_STALL_NOTIF) + add_definitions(-DROCKSDB_DISABLE_STALL_NOTIFICATION) +endif() + + +if(DEFINED USE_RTTI) + if(USE_RTTI) + message(STATUS "Enabling RTTI") + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DROCKSDB_USE_RTTI") + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DROCKSDB_USE_RTTI") + else() + if(MSVC) + message(STATUS "Disabling RTTI in Release builds. Always on in Debug.") + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DROCKSDB_USE_RTTI") + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /GR-") + else() + message(STATUS "Disabling RTTI in Release builds") + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fno-rtti") + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fno-rtti") + endif() + endif() +else() + message(STATUS "Enabling RTTI in Debug builds only (default)") + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DROCKSDB_USE_RTTI") + if(MSVC) + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /GR-") + else() + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fno-rtti") + endif() +endif() + +# Used to run CI build and tests so we can run faster +option(OPTDBG "Build optimized debug build with MSVC" OFF) +option(WITH_RUNTIME_DEBUG "build with debug version of runtime library" ON) +if(MSVC) + if(OPTDBG) + message(STATUS "Debug optimization is enabled") + set(CMAKE_CXX_FLAGS_DEBUG "/Oxt") + else() + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /Gm") + endif() + if(WITH_RUNTIME_DEBUG) + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /${RUNTIME_LIBRARY}d") + else() + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /${RUNTIME_LIBRARY}") + endif() + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Oxt /Zp8 /Gm- /Gy /${RUNTIME_LIBRARY}") + + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DEBUG") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG") +endif() + +if(CMAKE_COMPILER_IS_GNUCXX) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-builtin-memcmp") +endif() + +option(ROCKSDB_LITE "Build RocksDBLite version" OFF) +if(ROCKSDB_LITE) + add_definitions(-DROCKSDB_LITE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exceptions -Os") +endif() + +if(CMAKE_SYSTEM_NAME MATCHES "Cygwin") + add_definitions(-fno-builtin-memcmp -DCYGWIN) +elseif(CMAKE_SYSTEM_NAME MATCHES "Darwin") + add_definitions(-DOS_MACOSX) + if(CMAKE_SYSTEM_PROCESSOR MATCHES arm) + add_definitions(-DIOS_CROSS_COMPILE -DROCKSDB_LITE) + # no debug info for IOS, that will make our library big + add_definitions(-DNDEBUG) + endif() +elseif(CMAKE_SYSTEM_NAME MATCHES "Linux") + add_definitions(-DOS_LINUX) +elseif(CMAKE_SYSTEM_NAME MATCHES "SunOS") + add_definitions(-DOS_SOLARIS) +elseif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") + add_definitions(-DOS_FREEBSD) +elseif(CMAKE_SYSTEM_NAME MATCHES "NetBSD") + add_definitions(-DOS_NETBSD) +elseif(CMAKE_SYSTEM_NAME MATCHES "OpenBSD") + add_definitions(-DOS_OPENBSD) +elseif(CMAKE_SYSTEM_NAME MATCHES "DragonFly") + add_definitions(-DOS_DRAGONFLYBSD) +elseif(CMAKE_SYSTEM_NAME MATCHES "Android") + add_definitions(-DOS_ANDROID) +elseif(CMAKE_SYSTEM_NAME MATCHES "Windows") + add_definitions(-DWIN32 -DOS_WIN -D_MBCS -DWIN64 -DNOMINMAX) + if(MINGW) + add_definitions(-D_WIN32_WINNT=_WIN32_WINNT_VISTA) + endif() +endif() + +if(NOT WIN32) + add_definitions(-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX) +endif() + +option(WITH_FALLOCATE "build with fallocate" ON) +if(WITH_FALLOCATE) + CHECK_CXX_SOURCE_COMPILES(" +#include +#include +int main() { + int fd = open(\"/dev/null\", 0); + fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, 1024); +} +" HAVE_FALLOCATE) + if(HAVE_FALLOCATE) + add_definitions(-DROCKSDB_FALLOCATE_PRESENT) + endif() +endif() + +CHECK_CXX_SOURCE_COMPILES(" +#include +int main() { + int fd = open(\"/dev/null\", 0); + sync_file_range(fd, 0, 1024, SYNC_FILE_RANGE_WRITE); +} +" HAVE_SYNC_FILE_RANGE_WRITE) +if(HAVE_SYNC_FILE_RANGE_WRITE) + add_definitions(-DROCKSDB_RANGESYNC_PRESENT) +endif() + +CHECK_CXX_SOURCE_COMPILES(" +#include +int main() { + (void) PTHREAD_MUTEX_ADAPTIVE_NP; +} +" HAVE_PTHREAD_MUTEX_ADAPTIVE_NP) +if(HAVE_PTHREAD_MUTEX_ADAPTIVE_NP) + add_definitions(-DROCKSDB_PTHREAD_ADAPTIVE_MUTEX) +endif() + +include(CheckCXXSymbolExists) +check_cxx_symbol_exists(malloc_usable_size malloc.h HAVE_MALLOC_USABLE_SIZE) +if(HAVE_MALLOC_USABLE_SIZE) + add_definitions(-DROCKSDB_MALLOC_USABLE_SIZE) +endif() + +check_cxx_symbol_exists(sched_getcpu sched.h HAVE_SCHED_GETCPU) +if(HAVE_SCHED_GETCPU) + add_definitions(-DROCKSDB_SCHED_GETCPU_PRESENT) +endif() \ No newline at end of file diff --git a/librocksdb_sys/librocksdb_cloud_sys/rocksdb-cloud b/librocksdb_sys/librocksdb_cloud_sys/rocksdb-cloud new file mode 160000 index 000000000..ac0e18f44 --- /dev/null +++ b/librocksdb_sys/librocksdb_cloud_sys/rocksdb-cloud @@ -0,0 +1 @@ +Subproject commit ac0e18f441de3e265c9f15d69dec22220f916814 diff --git a/librocksdb_sys/librocksdb_cloud_sys/src/lib.rs b/librocksdb_sys/librocksdb_cloud_sys/src/lib.rs new file mode 100644 index 000000000..0209009e3 --- /dev/null +++ b/librocksdb_sys/librocksdb_cloud_sys/src/lib.rs @@ -0,0 +1 @@ +// The library requires this file for `cargo build` diff --git a/librocksdb_sys/rocksdb b/librocksdb_sys/rocksdb index fa9b061b8..f114e1361 160000 --- a/librocksdb_sys/rocksdb +++ b/librocksdb_sys/rocksdb @@ -1 +1 @@ -Subproject commit fa9b061b8717cbecbae370762c097b41f32107aa +Subproject commit f114e1361188c41f7612ba7ee575d02b88dbd80c diff --git a/librocksdb_sys/src/lib.rs b/librocksdb_sys/src/lib.rs index 5cd193ea1..7770e26f0 100644 --- a/librocksdb_sys/src/lib.rs +++ b/librocksdb_sys/src/lib.rs @@ -46,6 +46,8 @@ use libc::{c_char, c_double, c_int, c_uchar, c_void, size_t}; #[repr(C)] pub struct Options(c_void); #[repr(C)] +pub struct CloudEnvOptions(c_void); +#[repr(C)] pub struct ColumnFamilyDescriptor(c_void); #[repr(C)] pub struct DBInstance(c_void); @@ -2318,6 +2320,26 @@ extern "C" { ); } +// RocksDB Cloud +extern "C" { + // NewAWSEnv + pub fn crocksdb_cloud_aws_env_create( + base_env: *mut DBEnv, + src_cloud_bucket: *const c_char, + src_cloud_object: *const c_char, + src_cloud_region: *const c_char, + dest_cloud_bucket: *const c_char, + dest_cloud_object: *const c_char, + dest_cloud_region: *const c_char, + opts: *mut CloudEnvOptions, + err: *mut *mut c_char, + ) -> *mut DBEnv; + + // CloudEnvOptions + pub fn crocksdb_cloud_envoptions_create() -> *mut CloudEnvOptions; + pub fn crocksdb_cloud_envoptions_destroy(opt: *mut CloudEnvOptions); +} + #[cfg(test)] mod test { use super::*; diff --git a/src/cloud.rs b/src/cloud.rs new file mode 100644 index 000000000..e57e38f5a --- /dev/null +++ b/src/cloud.rs @@ -0,0 +1,26 @@ +// Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0. + +use crocksdb_ffi::{self}; + +/// Options while opening a file in the cloud to read/write +pub struct CloudEnvOptions { + pub inner: *mut crocksdb_ffi::CloudEnvOptions, +} + +impl CloudEnvOptions { + pub fn new() -> CloudEnvOptions { + unsafe { + CloudEnvOptions { + inner: crocksdb_ffi::crocksdb_cloud_envoptions_create(), + } + } + } +} + +impl Drop for CloudEnvOptions { + fn drop(&mut self) { + unsafe { + crocksdb_ffi::crocksdb_cloud_envoptions_destroy(self.inner); + } + } +} diff --git a/src/lib.rs b/src/lib.rs index f67b75ad2..9205dd51b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -24,6 +24,8 @@ pub extern crate librocksdb_sys; #[cfg(test)] extern crate tempfile; +#[cfg(feature = "cloud")] +pub use cloud::CloudEnvOptions; pub use compaction_filter::{ new_compaction_filter, new_compaction_filter_factory, new_compaction_filter_raw, CompactionFilter, CompactionFilterContext, CompactionFilterFactory, @@ -69,6 +71,8 @@ pub use titan::{TitanBlobIndex, TitanDBOptions}; #[allow(deprecated)] pub use rocksdb::Kv; +#[cfg(feature = "cloud")] +mod cloud; mod compaction_filter; pub mod comparator; #[cfg(feature = "encryption")] diff --git a/src/rocksdb.rs b/src/rocksdb.rs index 677e99c82..32dd1556e 100644 --- a/src/rocksdb.rs +++ b/src/rocksdb.rs @@ -39,6 +39,8 @@ use std::str::from_utf8; use std::sync::Arc; use std::{fs, ptr, slice}; +#[cfg(feature = "cloud")] +use cloud::CloudEnvOptions; #[cfg(feature = "encryption")] use encryption::{DBEncryptionKeyManager, EncryptionKeyManager}; use table_properties::{TableProperties, TablePropertiesCollection}; @@ -2543,6 +2545,47 @@ impl Env { } } + // Create an cloud env to operate with AWS S3. + #[cfg(feature = "cloud")] + pub fn new_aws_env( + base_env: Arc, + src_cloud_bucket: &str, + src_cloud_object: &str, + src_cloud_region: &str, + dest_cloud_bucket: &str, + dest_cloud_object: &str, + dest_cloud_region: &str, + opts: CloudEnvOptions, + ) -> Result { + let mut err = ptr::null_mut(); + let src_cloud_bucket = CString::new(src_cloud_bucket).unwrap(); + let src_cloud_object = CString::new(src_cloud_object).unwrap(); + let src_cloud_region = CString::new(src_cloud_region).unwrap(); + let dest_cloud_bucket = CString::new(dest_cloud_bucket).unwrap(); + let dest_cloud_object = CString::new(dest_cloud_object).unwrap(); + let dest_cloud_region = CString::new(dest_cloud_region).unwrap(); + let env = unsafe { + crocksdb_ffi::crocksdb_cloud_aws_env_create( + base_env.inner, + src_cloud_bucket.as_ptr(), + src_cloud_object.as_ptr(), + src_cloud_region.as_ptr(), + dest_cloud_bucket.as_ptr(), + dest_cloud_object.as_ptr(), + dest_cloud_region.as_ptr(), + opts.inner, + &mut err, + ) + }; + if !err.is_null() { + return Err(unsafe { crocksdb_ffi::error_message(err) }); + } + Ok(Env { + inner: env, + base: Some(base_env), + }) + } + // Create a ctr encrypted env with a given base env and a given ciper text. // The length of ciper text must be 2^n, and must be less or equal to 2048. // The recommanded block size are 1024, 512 and 256. @@ -3444,4 +3487,22 @@ mod test { let first_path = db.get_db_options().get_db_path(0).unwrap(); assert_eq!(path, first_path.as_str()); } + + #[cfg(feature = "cloud")] + #[test] + fn test_cloud_aws_env_creation() { + let k_db_path = "/tmp/rocksdb_main_db"; + let k_bucket_suffix = "cloud.clone.example."; + let k_region = "us-west-2"; + let _db = Env::new_aws_env( + Arc::new(Env::default()), + &k_bucket_suffix, + &k_db_path, + &k_region, + &k_bucket_suffix, + &k_db_path, + &k_region, + CloudEnvOptions::new(), + ); + } } diff --git a/tests/cases/test_rocksdb_options.rs b/tests/cases/test_rocksdb_options.rs index c7fd5ef64..1ed280986 100644 --- a/tests/cases/test_rocksdb_options.rs +++ b/tests/cases/test_rocksdb_options.rs @@ -175,7 +175,7 @@ fn test_set_ratelimiter_with_auto_tuned() { 100 * 1024 * 1024, 10 * 100000, DBRateLimiterMode::AllIo, - true + true, ); let db = DB::open(opts, path.path().to_str().unwrap()).unwrap(); drop(db);