JNI Reader for Table Iterator

include/rocksdb/sst_file_reader.h

@@ -35,6 +35,15 @@ class SstFileReader {
                                const std::vector<Slice>& keys,
                                std::vector<std::string>* values);
 
+  // Returns a new iterator over the table contents as a raw table iterator,
+  // a.k.a a `TableIterator`that iterates all point data entries in the table
+  // including logically invisible entries like delete entries lying in a range inclusive of from_key and
+  // exclusive of to_key.
+  // This API is intended to provide a programmatic way to observe SST files
+  // created by a DB, to be used by third party tools. DB optimization
+  // capabilities like filling cache, read ahead are disabled.
+  std::unique_ptr<Iterator> NewTableIterator(const Slice* from_key, const Slice* to_key);
+
   // Returns a new iterator over the table contents as a raw table iterator,
   // a.k.a a `TableIterator`that iterates all point data entries in the table
   // including logically invisible entries like delete entries.

include/rocksdb/types.h

@@ -81,6 +81,7 @@ struct ParsedEntryInfo {
   Slice timestamp;
   SequenceNumber sequence;
   EntryType type;
+  bool copied_user_key;
 };
 
 enum class WriteStallCause {

include/rocksdb/utilities/types_util.h

@@ -27,6 +27,12 @@ Status GetInternalKeyForSeekForPrev(const Slice& user_key,
                                     const Comparator* comparator,
                                     std::string* buf);
 
+// Util method that takes an internal key and parse it to get `ParsedEntryInfo`.
+// Such an internal key usually comes from a table iterator.
+// `comparator`: see doc for `GetInternalKeyForSeek`.
+Status ParseEntry(const Slice& internal_key, const Comparator* comparator,
+                  ParsedEntryInfo* parsed_entry, bool copied_user_key);
+
 // Util method that takes an internal key and parse it to get `ParsedEntryInfo`.
 // Such an internal key usually comes from a table iterator.
 // `comparator`: see doc for `GetInternalKeyForSeek`.

java/CMakeLists.txt

@@ -99,6 +99,8 @@ set(JNI_NATIVE_SOURCES
         rocksjni/write_batch_with_index.cc
         rocksjni/write_buffer_manager.cc
         rocksjni/writebatchhandlerjnicallback.cc
+        rocksjni/parsed_entry_info.cc
+        rocksjni/type_util.cc
 )
 
 set(JAVA_MAIN_CLASSES
@@ -308,6 +310,9 @@ set(JAVA_MAIN_CLASSES
   src/test/java/org/rocksdb/test/TestableEventListener.java
   src/test/java/org/rocksdb/util/CapturingWriteBatchHandler.java
   src/test/java/org/rocksdb/util/WriteBatchGetter.java
+  src/main/java/org/rocksdb/ParsedEntryInfo.java
+  src/main/java/org/rocksdb/EntryType.java
+  src/main/java/org/rocksdb/TypeUtil.java
 )
 
 set(JAVA_TEST_CLASSES

java/rocksjni/parsed_entry_info.cc

@@ -0,0 +1,199 @@
+// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+//
+// This file implements the "bridge" between Java and C++ and enables
+// calling c++ ROCKSDB_NAMESPACE::Iterator methods from Java side.
+
+#include <jni.h>
+#include <stdlib.h>
+
+#include "include/org_rocksdb_ParsedEntryInfo.h"
+#include "rocksdb/options.h"
+#include "rocksdb/types.h"
+#include "rocksdb/utilities/types_util.h"
+#include "rocksjni/cplusplus_to_java_convert.h"
+#include "rocksjni/portal.h"
+
+/*
+ * Class:     org_rocksdb_ParsedEntryInfo
+ * Method:    newParseEntryInstance
+ * Signature: ()J
+ */
+jlong JNICALL Java_org_rocksdb_ParsedEntryInfo_newParseEntryInstance(
+    JNIEnv * /*env*/, jclass /*cls*/) {
+  ROCKSDB_NAMESPACE::ParsedEntryInfo *parsed_entry_info =
+      new ROCKSDB_NAMESPACE::ParsedEntryInfo();
+  return GET_CPLUSPLUS_POINTER(parsed_entry_info);
+}
+
+/*
+ * Class:     org_rocksdb_ParsedEntryInfo
+ * Method:    parseEntry
+ * Signature: (JJ[BI)V
+ */
+void JNICALL Java_org_rocksdb_ParsedEntryInfo_parseEntry(
+    JNIEnv *env, jclass /*cls*/, jlong handle, jlong options_handle,
+    jbyteArray jtarget, jint len) {
+  auto *options =
+      reinterpret_cast<const ROCKSDB_NAMESPACE::Options *>(options_handle);
+  auto *parsed_entry_info =
+      reinterpret_cast<ROCKSDB_NAMESPACE::ParsedEntryInfo *>(handle);
+  jbyte* target = new jbyte[len];
+  env->GetByteArrayRegion(jtarget, 0, len, target);
+  if (env->ExceptionCheck()) {
+    return;
+  }
+  ROCKSDB_NAMESPACE::Slice target_slice(reinterpret_cast<char *>(target), len);
+  ROCKSDB_NAMESPACE::Status s = ROCKSDB_NAMESPACE::ParseEntry(target_slice, options->comparator,
+                                parsed_entry_info, true);
+  if (!s.ok()) {
+    ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s);
+  }
+}
+
+/*
+ * Class:     org_rocksdb_ParsedEntryInfo
+ * Method:    parseEntryDirect
+ * Signature: (JJLjava/nio/ByteBuffer;II)V
+ */
+void JNICALL Java_org_rocksdb_ParsedEntryInfo_parseEntryDirect(
+    JNIEnv *env, jclass /*clz*/, jlong handle, jlong options_handle,
+    jobject jbuffer, jint jbuffer_off, jint jbuffer_len) {
+  auto *options =
+      reinterpret_cast<const ROCKSDB_NAMESPACE::Options *>(options_handle);
+  auto *parsed_entry_info =
+      reinterpret_cast<ROCKSDB_NAMESPACE::ParsedEntryInfo *>(handle);
+  auto parse = [&env, &parsed_entry_info,
+                &options](ROCKSDB_NAMESPACE::Slice &target_slice) {
+    ROCKSDB_NAMESPACE::Status s = ROCKSDB_NAMESPACE::ParseEntry(target_slice, options->comparator,
+                                  parsed_entry_info);
+    if (!s.ok()) {
+      ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s);
+      return;
+    }
+  };
+  ROCKSDB_NAMESPACE::JniUtil::k_op_direct(parse, env, jbuffer, jbuffer_off,
+                                          jbuffer_len);
+}
+
+/*
+ * Class:     org_rocksdb_ParsedEntryInfo
+ * Method:    parseEntryByteArray
+ * Signature: (JJ[BII)V
+ */
+void JNICALL Java_org_rocksdb_ParsedEntryInfo_parseEntryByteArray(
+    JNIEnv *env, jclass /*clz*/, jlong handle, jlong options_handle,
+    jbyteArray jtarget, jint joff, jint jlen) {
+  auto *options =
+      reinterpret_cast<const ROCKSDB_NAMESPACE::Options *>(options_handle);
+  auto *parsed_entry_info =
+      reinterpret_cast<ROCKSDB_NAMESPACE::ParsedEntryInfo *>(handle);
+  auto parse = [&env, &parsed_entry_info,
+                &options](ROCKSDB_NAMESPACE::Slice &target_slice) {
+    ROCKSDB_NAMESPACE::Status s = ROCKSDB_NAMESPACE::ParseEntry(target_slice, options->comparator,
+                                  parsed_entry_info);
+    if (!s.ok()) {
+      ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, s);
+      return true;
+    }
+    return false;
+  };
+  ROCKSDB_NAMESPACE::JniUtil::k_op_indirect(parse, env, jtarget, joff, jlen);
+}
+
+/*
+ * Class:     org_rocksdb_ParsedEntryInfo
+ * Method:    userKeyDirect
+ * Signature: (JLjava/nio/ByteBuffer;II)I
+ */
+jint JNICALL Java_org_rocksdb_ParsedEntryInfo_userKeyDirect(
+    JNIEnv *env, jclass /*clz*/, jlong handle, jobject jtarget, jint joffset,
+    jint jlen) {
+  auto *parsed_entry_info =
+      reinterpret_cast<ROCKSDB_NAMESPACE::ParsedEntryInfo *>(handle);
+  ROCKSDB_NAMESPACE::Slice key_slice = parsed_entry_info->user_key;
+  return ROCKSDB_NAMESPACE::JniUtil::copyToDirect(env, key_slice, jtarget,
+                                                  joffset, jlen);
+}
+
+/*
+ * Class:     org_rocksdb_ParsedEntryInfo
+ * Method:    userKeyByteArray
+ * Signature: (J[BII)I
+ */
+jint JNICALL Java_org_rocksdb_ParsedEntryInfo_userKeyByteArray(
+    JNIEnv *env, jclass /*clz*/, jlong handle, jbyteArray jtarget, jint joffset,
+    jint jlen) {
+  auto *parsed_entry_info =
+      reinterpret_cast<ROCKSDB_NAMESPACE::ParsedEntryInfo *>(handle);
+  ROCKSDB_NAMESPACE::Slice key_slice = parsed_entry_info->user_key;
+  return ROCKSDB_NAMESPACE::JniUtil::copyBytes(env, key_slice, jtarget, joffset,
+                                               jlen);
+}
+
+/*
+ * Class:     org_rocksdb_ParsedEntryInfo
+ * Method:    userKeyJni
+ * Signature: (J)[B
+ */
+jbyteArray JNICALL Java_org_rocksdb_ParsedEntryInfo_userKeyJni(JNIEnv *env,
+                                                               jclass /*clz*/,
+                                                               jlong handle) {
+  auto *parsed_entry_info =
+      reinterpret_cast<ROCKSDB_NAMESPACE::ParsedEntryInfo *>(handle);
+  ROCKSDB_NAMESPACE::Slice key_slice = parsed_entry_info->user_key;
+  jbyteArray jkey = env->NewByteArray(static_cast<jsize>(key_slice.size()));
+  if (jkey == nullptr) {
+    // exception thrown: OutOfMemoryError
+    ROCKSDB_NAMESPACE::OutOfMemoryErrorJni::ThrowNew(env, "Memory allocation failed in RocksDB JNI function");
+    return nullptr;
+  }
+  ROCKSDB_NAMESPACE::JniUtil::copyBytes(env, key_slice, jkey, 0,
+                                        static_cast<jint>(key_slice.size()));
+  return jkey;
+}
+
+/*
+ * Class:     org_rocksdb_ParsedEntryInfo
+ * Method:    getSequenceNumberJni
+ * Signature: (J)J
+ */
+jlong JNICALL Java_org_rocksdb_ParsedEntryInfo_getSequenceNumberJni(
+    JNIEnv * /*env*/, jclass /*clz*/, jlong handle) {
+  auto *parsed_entry_info =
+      reinterpret_cast<ROCKSDB_NAMESPACE::ParsedEntryInfo *>(handle);
+  uint64_t sequence_number = parsed_entry_info->sequence;
+  return static_cast<jlong>(sequence_number);
+}
+
+/*
+ * Class:     org_rocksdb_ParsedEntryInfo
+ * Method:    getValueTypeJni
+ * Signature: (J)B
+ */
+jbyte JNICALL Java_org_rocksdb_ParsedEntryInfo_getEntryTypeJni(JNIEnv * /*env*/,
+                                                               jclass /*clz*/,
+                                                               jlong handle) {
+  auto *parsed_entry_info =
+      reinterpret_cast<ROCKSDB_NAMESPACE::ParsedEntryInfo *>(handle);
+  ROCKSDB_NAMESPACE::EntryType type = parsed_entry_info->type;
+  return ROCKSDB_NAMESPACE::EntryTypeJni::toJavaEntryType(type);
+}
+
+/*
+ * Class:     org_rocksdb_ParsedEntryInfo
+ * Method:    disposeInternalJni
+ * Signature: (J)V
+ */
+void JNICALL Java_org_rocksdb_ParsedEntryInfo_disposeInternalJni(
+    JNIEnv * /*env*/, jclass /*clz*/, jlong handle) {
+  auto *parsed_entry_info =
+      reinterpret_cast<ROCKSDB_NAMESPACE::ParsedEntryInfo *>(handle);
+  assert(parsed_entry_info != nullptr);
+  if (parsed_entry_info->copied_user_key) {
+    delete parsed_entry_info->user_key.data();
+  }
+  delete parsed_entry_info;
+}

java/rocksjni/portal.h

@@ -24,6 +24,7 @@
 #include <type_traits>
 #include <vector>
 
+#include "db/dbformat.h"
 #include "rocksdb/convenience.h"
 #include "rocksdb/db.h"
 #include "rocksdb/filter_policy.h"
@@ -2477,6 +2478,42 @@ class JniUtil {
     return op(key_slice);
   }
 
+  /*
+   * Helper for operations on a key on java byte array
+   * Copies values from jbyte array to slice and performs op on the slice.
+   * for example WriteBatch->Delete
+   *
+   * from `op` and used for RocksDB->Delete etc.
+   */
+  static void k_op_indirect(std::function<bool(ROCKSDB_NAMESPACE::Slice&)> op,
+                            JNIEnv* env, jbyteArray jkey, jint jkey_off,
+                            jint jkey_len) {
+    if (jkey == nullptr || env->GetArrayLength(jkey) < (jkey_off + jkey_len)) {
+      ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env,
+                                                       "Invalid key argument");
+      return;
+    }
+    char* target = new char[jkey_len];
+    if (target == nullptr) {
+      ROCKSDB_NAMESPACE::OutOfMemoryErrorJni::ThrowNew(env,
+           "Memory allocation failed in RocksDB JNI function");
+      return;
+    }
+    env->GetByteArrayRegion(jkey, jkey_off, jkey_len,
+                            reinterpret_cast<jbyte*>(target));
+    if (env->ExceptionCheck()) {
+      // exception thrown: ArrayIndexOutOfBoundsException
+      ROCKSDB_NAMESPACE::IllegalArgumentExceptionJni::ThrowNew(env,
+           "Failed to get byte array region: Out of bounds or invalid array.");
+      return;
+    }
+    ROCKSDB_NAMESPACE::Slice target_slice(target, jkey_len);
+    bool release_copy = op(target_slice);
+    if (release_copy) {
+      delete[] target;
+    }
+  }
+
   template <class T>
   static jint copyToDirect(JNIEnv* env, T& source, jobject jtarget,
                            jint jtarget_off, jint jtarget_len) {
@@ -2498,6 +2535,31 @@ class JniUtil {
 
     return cvalue_len;
   }
+
+  /* Helper for copying value in source into a byte array.
+   */
+  template <class T>
+  static jint copyBytes(JNIEnv* env, T& source, jbyteArray jtarget,
+                        jint jtarget_off, jint jtarget_len) {
+    if (jtarget == nullptr ||
+        env->GetArrayLength(jtarget) < (jtarget_off + jtarget_len)) {
+      ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(
+          env, "Invalid target argument");
+      return 0;
+    }
+
+    const jint cvalue_len = static_cast<jint>(source.size());
+    const jint length = std::min(jtarget_len, cvalue_len);
+    env->SetByteArrayRegion(
+        jtarget, jtarget_off, length,
+        const_cast<jbyte*>(reinterpret_cast<const jbyte*>(source.data())));
+    if (env->ExceptionCheck()) {
+      ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(
+          env, "Failed to copy data to byte array.");
+      return 0;
+    }
+    return cvalue_len;
+  }
 };
 
 class MapJni : public JavaClass {
@@ -6444,6 +6506,35 @@ class TxnDBWritePolicyJni {
   }
 };
 
+// The portal class for org.rocksdb.EntryType
+class EntryTypeJni {
+ public:
+  static jbyte toJavaEntryType(const ROCKSDB_NAMESPACE::EntryType& entry_type) {
+    switch (entry_type) {
+      case ROCKSDB_NAMESPACE::EntryType::kEntryPut:
+        return 0x0;
+      case ROCKSDB_NAMESPACE::EntryType::kEntryDelete:
+        return 0x1;
+      case ROCKSDB_NAMESPACE::EntryType::kEntrySingleDelete:
+        return 0x2;
+      case ROCKSDB_NAMESPACE::EntryType::kEntryMerge:
+        return 0x3;
+      case ROCKSDB_NAMESPACE::EntryType::kEntryRangeDeletion:
+        return 0x4;
+      case ROCKSDB_NAMESPACE::EntryType::kEntryBlobIndex:
+        return 0x5;
+      case ROCKSDB_NAMESPACE::EntryType::kEntryDeleteWithTimestamp:
+        return 0x6;
+      case ROCKSDB_NAMESPACE::EntryType::kEntryWideColumnEntity:
+        return 0x7;
+      case ROCKSDB_NAMESPACE::EntryType::kEntryOther:
+        return 0x8;
+      default:
+        return 0x9;
+    }
+  }
+};
+
 // The portal class for org.rocksdb.TransactionDB.KeyLockInfo
 class KeyLockInfoJni : public JavaClass {
  public: