Add support for cloud data store

.gitmodules

@@ -2,8 +2,13 @@
 	path = librocksdb_sys/rocksdb
 	url = https://github.com/tikv/rocksdb.git
 	branch = 6.4.tikv
+
 [submodule "titan"]
 	path = librocksdb_sys/libtitan_sys/titan
 	url = https://github.com/tikv/titan.git
 	branch = master
 
+[submodule "rocksdb-cloud"]
+	path = librocksdb_sys/librocksdb_cloud_sys/rocksdb-cloud
+	url = https://github.com/tikv/rocksdb
+	branch = 6.4.cloud

.travis.yml

@@ -1,12 +1,14 @@
 dist: bionic
 language: rust
 
-os:
-  - linux
-
 rust:
   - nightly
 
+cache:
+  cargo: false
+  directories:
+  - "/tmp/aws-sdk-cpp-1.8.14"
+
 addons:
   apt:
     sources:
@@ -19,6 +21,9 @@ jobs:
   - os: linux
     rust: nightly
     env: FEATURES="encryption,jemalloc,portable,sse"
+  - os: linux
+    rust: nightly
+    env: FEATURES="cloud"  
   - os: linux
     arch: arm64
     rust: nightly
@@ -31,14 +36,26 @@ env:
   global:
   - RUST_BACKTRACE=1
 
-cache: false
-
 before_script:
+  # Installing the AWS-SDK, so increase the maximum number of open file descriptors,
+  # since some tests use more FDs than the default limit.
+  - if [ "${FEATURES}" == "cloud" ]; then
+      echo "limit maxfiles 1024 unlimited" | sudo tee -a /etc/launchd.conf;
+      pushd /tmp;
+      wget https://github.com/aws/aws-sdk-cpp/archive/1.8.14.tar.gz -O /tmp/aws-sdk.tar.gz;
+      tar -xvf /tmp/aws-sdk.tar.gz > /dev/null;
+      popd;
+      pushd /tmp/aws-sdk-cpp-1.8.14;
+      cmake -DBUILD_ONLY='kinesis;core;s3;transfer' -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_TESTING=OFF .;
+      make -j4 all;
+      sudo make install;
+      popd;
+    fi
   - rustup component add rustfmt-preview
   - rustup component add clippy
 
 script:
-  # copile rocksdb may cost more than 10 minutes, see https://docs.travis-ci.com/user/common-build-problems/#build-times-out-because-no-output-was-received
+  # compiling rocksdb may cost more than 10 minutes, see https://docs.travis-ci.com/user/common-build-problems/#build-times-out-because-no-output-was-received
   - if [ "${TASK}" == "clang-format" ]; then
       bash ./travis-build/travis-format.sh;
     else

Cargo.toml

@@ -22,6 +22,8 @@ portable = ["librocksdb_sys/portable"]
 sse = ["librocksdb_sys/sse"]
 static_libcpp = ["librocksdb_sys/static_libcpp"]
 valgrind = []
+# rocksdb-cloud integration
+cloud = ["librocksdb_sys/cloud"]
 
 [dependencies]
 libc = "0.2.11"

Makefile

@@ -41,3 +41,14 @@ update_rocksdb:
 	fi
 	@git submodule sync
 	@git submodule update --init --remote librocksdb_sys/rocksdb
+
+update_rocksdb_cloud:
+	@if [ -n "${ROCKSDB_CLOUD_REPO}" ]; then \
+		git config --file=.gitmodules submodule.rocksdb-cloud.url https://github.com/${ROCKSDB_CLOUD_REPO}/rocksdb.git; \
+	fi
+	@if [ -n "${ROCKSDB_CLOUD_BRANCH}" ]; then \
+		git config --file=.gitmodules submodule.rocksdb-cloud.branch ${ROCKSDB_CLOUD_BRANCH}; \
+	fi
+	@git submodule sync
+	@git submodule update --init --remote librocksdb_sys/librocksdb_cloud_sys/rocksdb-cloud
+

librocksdb_sys/Cargo.toml

@@ -8,6 +8,7 @@ links = "rocksdb"
 [dependencies]
 bzip2-sys = "0.1.8+1.0.8" 
 libc = "0.2.11"
+librocksdb_cloud_sys = { path = "librocksdb_cloud_sys" }
 libtitan_sys = { path = "libtitan_sys" }
 libz-sys = { version = "1.0.25", features = ["static"] }
 openssl-sys = { version = "0.9.54", optional = true, features = ["vendored"] }
@@ -25,6 +26,7 @@ jemalloc = ["tikv-jemalloc-sys"]
 portable = ["libtitan_sys/portable"]
 sse = ["libtitan_sys/sse"]
 static_libcpp = []
+cloud = []
 
 [build-dependencies]
 cc = "1.0.3"

librocksdb_sys/build.rs

@@ -128,6 +128,10 @@ fn link_cpp(build: &mut Build) {
 fn build_rocksdb() -> Build {
     let target = env::var("TARGET").expect("TARGET was not set");
     let mut cfg = Config::new("rocksdb");
+    // Conditionally compile with support for RocksDB-Cloud, setting USE_AWS
+    if cfg!(feature = "cloud") {
+        println!("cargo:rustc-link-lib=static=rocksdb_cloud");
+    }
     if cfg!(feature = "encryption") {
         cfg.register_dep("OPENSSL").define("WITH_OPENSSL", "ON");
         println!("cargo:rustc-link-lib=static=crypto");
@@ -195,13 +199,23 @@ fn build_rocksdb() -> Build {
     build.include(cur_dir.join("rocksdb"));
     build.include(cur_dir.join("libtitan_sys").join("titan").join("include"));
     build.include(cur_dir.join("libtitan_sys").join("titan"));
+    build.include(
+        cur_dir
+            .join("librocksdb_cloud_sys")
+            .join("rocksdb-cloud")
+            .join("include"),
+    );
+    build.include(cur_dir.join("librocksdb_cloud_sys").join("rocksdb-cloud"));
 
     // Adding rocksdb specific compile macros.
     // TODO: should make sure crocksdb compile options is the same as rocksdb and titan.
     build.define("ROCKSDB_SUPPORT_THREAD_LOCAL", None);
     if cfg!(feature = "encryption") {
         build.define("OPENSSL", None);
     }
+    if cfg!(feature = "cloud") {
+        build.define("USE_AWS", None).define("USE_CLOUD", None);
+    }
 
     println!("cargo:rustc-link-lib=static=rocksdb");
     println!("cargo:rustc-link-lib=static=titan");

librocksdb_sys/crocksdb/c.cc

@@ -15,6 +15,7 @@
 
 #include "db/column_family.h"
 #include "rocksdb/cache.h"
+#include "rocksdb/cloud/cloud_env_options.h"
 #include "rocksdb/compaction_filter.h"
 #include "rocksdb/comparator.h"
 #include "rocksdb/convenience.h"
@@ -178,6 +179,9 @@ using rocksdb::titandb::TitanDBOptions;
 using rocksdb::titandb::TitanOptions;
 using rocksdb::titandb::TitanReadOptions;
 
+using rocksdb::CloudEnv;
+using rocksdb::CloudEnvOptions;
+
 using rocksdb::MemoryAllocator;
 
 #ifdef OPENSSL
@@ -5853,4 +5857,49 @@ void ctitandb_delete_files_in_ranges_cf(
   SaveError(errptr, static_cast<TitanDB*>(db->rep)->DeleteFilesInRanges(
                         cf->rep, &ranges[0], num_ranges, include_end));
 }
+
+/* RocksDB Cloud */
+#ifdef USE_CLOUD
+struct crocksdb_cloud_envoptions_t {
+  CloudEnvOptions rep;
+};
+
+crocksdb_env_t* crocksdb_cloud_aws_env_create(
+    crocksdb_env_t* base_env, const char* src_cloud_bucket,
+    const char* src_cloud_object, const char* src_cloud_region,
+    const char* dest_cloud_bucket, const char* dest_cloud_object,
+    const char* dest_cloud_region, crocksdb_cloud_envoptions_t* cloud_options,
+    char** errptr) {
+  // Store a reference to a cloud env. A new cloud env object should be
+  // associated with every new cloud-db.
+  CloudEnv* cloud_env;
+
+  CloudEnv* cenv;
+  if (SaveError(errptr,
+                CloudEnv::NewAwsEnv(
+                    base_env->rep, src_cloud_bucket, src_cloud_object,
+                    src_cloud_region, dest_cloud_bucket, dest_cloud_object,
+                    dest_cloud_region, cloud_options->rep, nullptr, &cenv))) {
+    assert(cenv != nullptr);
+    return nullptr;
+  }
+  cloud_env = cenv;
+
+  crocksdb_env_t* result = new crocksdb_env_t;
+  result->rep = static_cast<Env*>(cloud_env);
+  result->block_cipher = nullptr;
+  result->encryption_provider = nullptr;
+  result->is_default = true;
+  return result;
+}
+
+crocksdb_cloud_envoptions_t* crocksdb_cloud_envoptions_create() {
+  crocksdb_cloud_envoptions_t* opt = new crocksdb_cloud_envoptions_t;
+  return opt;
+}
+
+void crocksdb_cloud_envoptions_destroy(crocksdb_cloud_envoptions_t* opt) {
+  delete opt;
+}
+#endif
 }  // end extern "C"

librocksdb_sys/crocksdb/crocksdb/c.h

@@ -70,6 +70,7 @@ extern "C" {
 
 /* Exported types */
 
+typedef struct crocksdb_cloud_envoptions_t crocksdb_cloud_envoptions_t;
 typedef struct crocksdb_t crocksdb_t;
 typedef struct crocksdb_status_ptr_t crocksdb_status_ptr_t;
 typedef struct crocksdb_backup_engine_t crocksdb_backup_engine_t;
@@ -2348,6 +2349,21 @@ extern C_ROCKSDB_LIBRARY_API void ctitandb_delete_files_in_ranges_cf(
     const char* const* limit_keys, const size_t* limit_keys_lens,
     size_t num_ranges, unsigned char include_end, char** errptr);
 
+/* RocksDB Cloud */
+
+#ifdef USE_CLOUD
+extern C_ROCKSDB_LIBRARY_API crocksdb_env_t* crocksdb_cloud_aws_env_create(
+    crocksdb_env_t* base_env, const char* src_cloud_bucket,
+    const char* src_cloud_object, const char* src_cloud_region,
+    const char* dest_cloud_bucket, const char* dest_cloud_object,
+    const char* dest_cloud_region, crocksdb_cloud_envoptions_t* cloud_options,
+    char** errptr);
+extern C_ROCKSDB_LIBRARY_API crocksdb_cloud_envoptions_t*
+crocksdb_cloud_envoptions_create();
+extern C_ROCKSDB_LIBRARY_API void crocksdb_cloud_envoptions_destroy(
+    crocksdb_cloud_envoptions_t* opt);
+#endif
+
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif

librocksdb_sys/librocksdb_cloud_sys/CMakeLists.txt

@@ -0,0 +1,60 @@
+# Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0.
+cmake_minimum_required(VERSION 3.0)
+project(rocksdb_cloud)
+enable_language(CXX)
+enable_language(C)
+
+# Manage absence of rocksdb code dependence
+
+if (NOT ROCKSDB_GIT_REPO)
+  set(ROCKSDB_GIT_REPO "https://github.com/tikv/rocksdb.git")
+endif()
+
+if (NOT ROCKSDB_GIT_BRANCH)
+  set(ROCKSDB_GIT_BRANCH "6.4.tikv")
+endif()
+
+if (NOT DEFINED ROCKSDB_DIR)
+  if (GIT_FOUND)
+    if (WIN32)
+      execute_process(COMMAND $ENV{COMSPEC} /C ${GIT_EXECUTABLE} clone --branch=${ROCKSDB_GIT_BRANCH} ${ROCKSDB_GIT_REPO})
+    else()
+      execute_process(COMMAND ${GIT_EXECUTABLE} clone --branch=${ROCKSDB_GIT_BRANCH} ${ROCKSDB_GIT_REPO})
+    endif()
+    set(ROCKSDB_DIR "${CMAKE_BINARY_DIR}/rocksdb")
+  endif()
+endif()
+
+if (NOT DEFINED ROCKSDB_DIR)
+  message(FATAL_ERROR "ROCKSDB_DIR is not defined.")
+endif()
+
+# Append rocksdb modules
+list(APPEND CMAKE_MODULE_PATH "${ROCKSDB_DIR}/cmake/modules/")
+include("cmake_modules/rocksdb_flags.cmake")
+
+include_directories("${ROCKSDB_DIR}")
+include_directories("${ROCKSDB_DIR}/include")
+include_directories("rocksdb-cloud")
+include_directories("rocksdb-cloud/include")
+
+file(GLOB SOURCES rocksdb-cloud/cloud/*.cc)
+
+add_library(rocksdb_cloud STATIC ${SOURCES} $<TARGET_OBJECTS:cloud_env>)
+
+option(WITH_CLOUD_TESTS "Build with tests." ON)
+
+include_directories(SYSTEM ${ROCKSDB_DIR}/third-party/gtest-1.7.0/fused-src)
+
+add_library(cloud_env OBJECT "rocksdb-cloud/cloud/cloud_env.cc")
+
+include(GNUInstallDirs)
+install(DIRECTORY include/rocksdb/cloud
+  COMPONENT devel
+  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
+)
+install(TARGETS rocksdb_cloud
+  COMPONENT devel
+  ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
+  INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
+)

librocksdb_sys/librocksdb_cloud_sys/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "librocksdb_cloud_sys"
+version = "0.1.0"
+edition = "2018"
+build = "build.rs"
+links = "rocksdb-cloud"
+
+[features]
+cloud = []
+
+[build-dependencies]
+cc = "1.0.3"
+cmake = "0.1"

librocksdb_sys/librocksdb_cloud_sys/build.rs

@@ -0,0 +1,18 @@
+// Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0.
+extern crate cc;
+extern crate cmake;
+
+use std::env;
+
+fn main() {
+    let cur_dir = env::current_dir().unwrap();
+    let mut cfg = cmake::Config::new(".");
+    let dst = cfg
+        .define("ROCKSDB_DIR", cur_dir.join("..").join("rocksdb"))
+        .env("USE_AWS", "1")
+        .build_target("rocksdb_cloud")
+        .very_verbose(true)
+        .build();
+    println!("cargo:rustc-link-search=native={}/build", dst.display());
+    println!("cargo:rustc-link-lib=static=rocksdb_cloud");
+}