Test PR

CMakeLists.txt

@@ -741,6 +741,7 @@ set(SOURCES
         utilities/debug.cc
         utilities/env_mirror.cc
         utilities/env_timed.cc
+        utilities/flink/flink_compaction_filter.cc
         utilities/leveldb_options/leveldb_options.cc
         utilities/memory/memory_util.cc
         utilities/merge_operators/bytesxor.cc
@@ -1122,6 +1123,7 @@ if(WITH_TESTS)
         utilities/cassandra/cassandra_format_test.cc
         utilities/cassandra/cassandra_row_merge_test.cc
         utilities/cassandra/cassandra_serialize_test.cc
+        utilities/flink/flink_compaction_filter_test.cc
         utilities/checkpoint/checkpoint_test.cc
         utilities/memory/memory_test.cc
         utilities/merge_operators/string_append/stringappend_test.cc

HISTORY.md

@@ -1,3 +1,11 @@
+# FRocksdb Change Log
+## 6.11.6-veverica-1.0 (10/22/2020)
+### Features
+* [Flink TTL] compaction filter for background cleanup of state with time-to-live
+* [Flink ListState] Separator-free merge operator
+### Bug FIxes
+* Fix a bug which may cause MultiGet to be slow because it may read more data than requested, but this won't affect correctness. The bug was introduced in 6.10 release.
+
 # Rocksdb Change Log
 ## 6.11.6 (10/12/2020)
 ### Bug Fixes

Makefile

@@ -6,6 +6,8 @@
 
 #-----------------------------------------------
 
+FROCKSDB_VERSION ?= 1.0
+
 BASH_EXISTS := $(shell which bash)
 SHELL := $(shell which bash)
 # Default to python3. Some distros like CentOS 8 do not have `python`.
@@ -585,6 +587,7 @@ TESTS = \
 	compaction_picker_test \
 	version_builder_test \
 	file_indexer_test \
+	flink_compaction_filter_test \
 	write_batch_test \
 	write_batch_with_index_test \
 	write_controller_test\
@@ -1407,6 +1410,9 @@ cassandra_serialize_test: utilities/cassandra/cassandra_serialize_test.o $(LIBOB
 hash_table_test: utilities/persistent_cache/hash_table_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
+flink_compaction_filter_test: utilities/flink/flink_compaction_filter_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 histogram_test: monitoring/histogram_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -2121,17 +2127,72 @@ rocksdbjavastatic: $(java_static_all_libobjects)
 		strip $(STRIPFLAGS) $(ROCKSDBJNILIB); \
 	fi
 	cd java;jar -cf target/$(ROCKSDB_JAR) HISTORY*.md
+	jar -uf java/target/$(ROCKSDB_JAR) HISTORY*.md
 	cd java/target;jar -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB)
 	cd java/target/classes;jar -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class
 	cd java/target/apidocs;jar -cf ../$(ROCKSDB_JAVADOCS_JAR) *
 	cd java/src/main/java;jar -cf ../../../target/$(ROCKSDB_SOURCES_JAR) org
 
+	mkdir -p java/target/META-INF
+	cp LICENSE.Apache java/target/META-INF/LICENSE
+	cd java/target;jar -uf $(ROCKSDB_JAR) META-INF/LICENSE
+
 rocksdbjavastaticrelease: rocksdbjavastatic
 	cd java/crossbuild && (vagrant destroy -f || true) && vagrant up linux32 && vagrant halt linux32 && vagrant up linux64 && vagrant halt linux64 && vagrant up linux64-musl && vagrant halt linux64-musl
 	cd java;jar -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md
-	cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
+	jar -uf java/target/$(ROCKSDB_JAR_ALL) HISTORY*.md
+	cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib librocksdbjni-win64.dll
 	cd java/target/classes;jar -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class
 
+frocksdbjavastaticrelease: rocksdbjavastaticrelease
+	# update license
+	mkdir -p java/target/META-INF
+	cp LICENSE.Apache java/target/META-INF/LICENSE
+	cd java/target;jar -uf $(ROCKSDB_JAR_ALL) META-INF/LICENSE
+
+	# platform jars
+	$(eval JAR_PREF=rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH))
+	$(eval JAR_DOCS=$(JAR_PREF)-javadoc.jar)
+	$(eval JAR_SOURCES=$(JAR_PREF)-sources.jar)
+	$(eval OSX_JAR=$(JAR_PREF)-osx.jar)
+	$(eval WIN_JAR=$(JAR_PREF)-win64.jar)
+	$(eval LINUX32_JAR=$(JAR_PREF)-linux32.jar)
+	$(eval LINUX64_JAR=$(JAR_PREF)-linux64.jar)
+
+	# update windows jar
+	cd java/target;cp rocksdbjni_classes.jar $(WIN_JAR)
+	cd java;jar -uf target/$(WIN_JAR) HISTORY*.md
+	jar -uf java/target/$(WIN_JAR) HISTORY*.md
+	cd java/target;jar -uf $(WIN_JAR) librocksdbjni-win64.dll
+	cd java/target;jar -uf $(WIN_JAR) META-INF/LICENSE
+
+	# update linux 64 jar with ppc64 lib
+	cd java/target;jar -uf $(LINUX64_JAR) librocksdbjni-linux-ppc64le.so
+
+	cd java/target;jar -uf $(JAR_DOCS) META-INF/LICENSE
+	cd java/target;jar -uf $(JAR_SOURCES) META-INF/LICENSE
+
+	# prepare frocksdb release
+	cd java/target;mkdir -p frocksdb-release
+
+	$(eval FJAR_PREF=frocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-artisans-$(FROCKSDB_VERSION))
+	$(eval FJAR=$(FJAR_PREF).jar)
+	$(eval FJAR_DOCS=$(FJAR_PREF)-javadoc.jar)
+	$(eval FJAR_SOURCES=$(FJAR_PREF)-sources.jar)
+	$(eval OSX_FJAR=$(FJAR_PREF)-osx.jar)
+	$(eval WIN_FJAR=$(FJAR_PREF)-win64.jar)
+	$(eval LINUX32_FJAR=$(FJAR_PREF)-linux32.jar)
+	$(eval LINUX64_FJAR=$(FJAR_PREF)-linux64.jar)
+
+	cd java/target;cp $(ROCKSDB_JAR_ALL) frocksdb-release/$(FJAR)
+	cd java/target;cp $(JAR_DOCS) frocksdb-release/$(FJAR_DOCS)
+	cd java/target;cp $(JAR_SOURCES) frocksdb-release/$(FJAR_SOURCES)
+	cd java/target;cp $(OSX_JAR) frocksdb-release/$(OSX_FJAR)
+	cd java/target;cp $(WIN_JAR) frocksdb-release/$(WIN_FJAR)
+	cd java/target;cp $(LINUX32_JAR) frocksdb-release/$(LINUX32_FJAR)
+	cd java/target;cp $(LINUX64_JAR) frocksdb-release/$(LINUX64_FJAR)
+	cd java;cp rocksjni.pom target/frocksdb-release/$(FJAR_PREF).pom
+
 rocksdbjavastaticreleasedocker: rocksdbjavastatic rocksdbjavastaticdockerx86 rocksdbjavastaticdockerx86_64 rocksdbjavastaticdockerx86musl rocksdbjavastaticdockerx86_64musl
 	cd java;jar -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md
 	cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib

RELEASE-FROCKSDB.md

@@ -0,0 +1,106 @@
+
+@@ -0,0 +1,104 @@
+## Build in Windows
+
+Use Windows 64 bit machine (e.g. base AWS Windows instance: 4 cores, 16GB RAM, 40GB storage for build).
+
+Install:
+ * git
+ * java 8
+ * maven
+ * Visual Studio Community 15 (2017)
+
+With [chocolatey](https://chocolatey.org/install):
+
+    choco install git.install jdk8 maven visualstudio2017community
+
+Optionally:
+
+    choco install intellijidea-community vscode
+
+Open developer command prompt for vs 2017 and run commands:
+
+    git clone [email protected]:dataArtisans/frocksdb.git
+    cd frocksdb
+    git checkout FRocksDB-6.11.6 # release branch
+    java\crossbuild\build-win.bat
+
+The result native library is `build\java\Release\rocksdbjni-shared.dll`.
+The result windows jar is `build\java\rocksdbjni_classes.jar`.
+
+There is also a how-to in CMakeLists.txt.
+
+## Build in PPC64LE
+
+Use Ubuntu 16.04 (e.g. AWS instance 4 cores, 16GB RAM, 40GB storage for build).
+Install git if not installed. If docker is installed, it might need to be removed.
+
+Setup ppc64le docker machine ([source](https://developer.ibm.com/linuxonpower/2017/06/08/build-test-ppc64le-docker-images-intel/)):
+
+    wget http://ftp.unicamp.br/pub/ppc64el/boot2docker/install.sh && chmod +x ./install.sh && ./install.sh -s
+    docker-machine create -d qemu \
+        --qemu-boot2docker-url=/home/ubuntu/.docker/machine/boot2docker.iso \
+        --qemu-memory 8192 \
+        --qemu-cache-mode none \
+        --qemu-arch ppc64le \
+        vm-ppc64le
+
+Regenerate certs as suggested if it did not work at once.
+
+Prepare docker machine to run rocksdbjni docker image for ppc64le build:
+
+    eval $(docker-machine env vm-ppc64le)
+    git clone [email protected]:dataArtisans/frocksdb.git
+    cd frocksdb
+    git checkout FRocksDB-6.11.6 # release branch
+    docker-machine ssh vm-ppc64le mkdir -p `pwd`
+    docker-machine scp -r . vm-ppc64le:`pwd`
+
+Build frocksdb:
+
+    make rocksdbjavastaticdockerppc64le
+    docker-machine scp vm-ppc64le:`pwd`/java/target/librocksdbjni-linux-ppc64le.so java/target/.
+
+The result native library is in `java/target/librocksdbjni-linux-ppc64le.so`.
+
+## Final crossbuild in Mac OSX
+
+Read how to Build cross jar for Mac OSX and linux as described in java/RELEASE.md but do not run it yet.
+
+Run commands:
+
+    make jclean clean
+    mkdir -p java/target
+    cp <path-to-windows-dll>/rocksdbjni-shared.dll java/target/librocksdbjni-win64.dll
+    cp <path-to-windows-jar>/rocksdbjni_classes.jar java/target/rocksdbjni_classes.jar
+    cp <path-to-ppc64le-lib-so>/librocksdbjni-linux-ppc64le.so java/target/librocksdbjni-linux-ppc64le.so
+    FROCKSDB_VERSION=1.0 make frocksdbjavastaticrelease
+
+## Push to maven central
+
+Edit the `frocksdbjni-<release version>.pom` file and replace 
+`<version>-</version>` 
+with 
+`<version><release version></version>` 
+where the `<release version>` is e.g. `6.11.6-veverica-1.0`.
+
+Run:
+```bash
+VERSION=<release version> \
+USER=<sonatype user> \
+PASSWORD=<sonatype password> \
+KEYNAME=<key name> \
+PASSPHRASE=<passphrase> \
+java/publish-frocksdbjni.sh
+```
+
+Go to the staging repositories on Sonatype:
+
+https://oss.sonatype.org/#stagingRepositories
+
+Select the open staging repository and click on "Close".
+
+Test the files in the staging repository 
+which will look something like this `https://oss.sonatype.org/content/repositories/comdata-artisans-1020`.
+
+Press the "Release" button (WARNING: this can not be undone).

TARGETS

@@ -1,4 +1,4 @@
-# This file @generated by `python3 buckifier/buckify_rocksdb.py`
+TARGETS# This file @generated by `python3 buckifier/buckify_rocksdb.py`
 # --> DO NOT EDIT MANUALLY <--
 # This file is a Facebook-specific integration for buck builds, so can
 # only be validated by Facebook employees.
@@ -330,6 +330,7 @@ cpp_library(
         "utilities/debug.cc",
         "utilities/env_mirror.cc",
         "utilities/env_timed.cc",
+        "utilities/flink/flink_compaction_filter.cc",
         "utilities/leveldb_options/leveldb_options.cc",
         "utilities/memory/memory_util.cc",
         "utilities/merge_operators/bytesxor.cc",
@@ -390,6 +391,7 @@ cpp_library(
         "tools/block_cache_analyzer/block_cache_trace_analyzer.cc",
         "tools/trace_analyzer_tool.cc",
         "utilities/cassandra/test_utils.cc",
+        "utilities/flink/flink_compaction_filter.cc",
     ],
     auto_headers = AutoHeaders.RECURSIVE_GLOB,
     arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
@@ -1106,6 +1108,11 @@ ROCKS_TESTS = [
         [],
         [],
     ],
+    [
+            "flink_compaction_filter_test",
+            "utilities/flink/flink_compaction_filter_test.cc",
+            "serial",
+    ],
     [
         "flush_job_test",
         "db/flush_job_test.cc",

Vagrantfile

@@ -2,7 +2,7 @@
 Vagrant.configure("2") do |config|
 
   config.vm.provider "virtualbox" do |v|
-    v.memory = 4096
+    v.memory = 6096
     v.cpus = 2
   end
 

file/random_access_file_reader.cc

@@ -194,7 +194,7 @@ bool TryMerge(FSReadRequest* dest, const FSReadRequest& src) {
   size_t src_offset = static_cast<size_t>(src.offset);
   size_t dest_end = End(*dest);
   size_t src_end = End(src);
-  if (std::max(dest_offset, dest_offset) > std::min(dest_end, src_end)) {
+  if (std::max(dest_offset, src_offset) > std::min(dest_end, src_end)) {
     return false;
   }
   dest->offset = static_cast<uint64_t>(std::min(dest_offset, src_offset));

java/CMakeLists.txt

@@ -29,6 +29,7 @@ set(JNI_NATIVE_SOURCES
         rocksjni/config_options.cc
         rocksjni/env.cc
         rocksjni/env_options.cc
+        rocksjni/flink_compactionfilterjni.cc
         rocksjni/filter.cc
         rocksjni/ingest_external_file_options.cc
         rocksjni/iterator.cc
@@ -140,6 +141,7 @@ set(JAVA_MAIN_CLASSES
   src/main/java/org/rocksdb/EnvOptions.java
   src/main/java/org/rocksdb/Experimental.java
   src/main/java/org/rocksdb/Filter.java
+  src/main/java/org/rocksdb/FlinkCompactionFilter.java
   src/main/java/org/rocksdb/FlushOptions.java
   src/main/java/org/rocksdb/HashLinkedListMemTableConfig.java
   src/main/java/org/rocksdb/HashSkipListMemTableConfig.java
@@ -211,6 +213,7 @@ set(JAVA_MAIN_CLASSES
   src/main/java/org/rocksdb/StatsLevel.java
   src/main/java/org/rocksdb/Status.java
   src/main/java/org/rocksdb/StringAppendOperator.java
+  src/main/java/org/rocksdb/StringAppendOperatorWithVariableDelimitor.java
   src/main/java/org/rocksdb/TableFilter.java
   src/main/java/org/rocksdb/TableProperties.java
   src/main/java/org/rocksdb/TableFormatConfig.java
@@ -415,6 +418,7 @@ if(${CMAKE_VERSION} VERSION_LESS "3.11.4" OR (${Java_VERSION_MINOR} STREQUAL "7"
           org.rocksdb.Env
           org.rocksdb.EnvOptions
           org.rocksdb.Filter
+          org.rocksdb.FlinkCompactionFilter
           org.rocksdb.FlushOptions
           org.rocksdb.HashLinkedListMemTableConfig
           org.rocksdb.HashSkipListMemTableConfig
@@ -453,6 +457,7 @@ if(${CMAKE_VERSION} VERSION_LESS "3.11.4" OR (${Java_VERSION_MINOR} STREQUAL "7"
           org.rocksdb.SstFileReaderIterator
           org.rocksdb.Statistics
           org.rocksdb.StringAppendOperator
+          org.rocksdb.StringAppendOperatorWithVariableDelimitor
           org.rocksdb.TableFormatConfig
           org.rocksdb.ThreadStatus
           org.rocksdb.TimedEnv

java/Makefile

@@ -30,6 +30,7 @@ NATIVE_JAVA_CLASSES = \
 	org.rocksdb.DirectSlice\
 	org.rocksdb.Env\
 	org.rocksdb.EnvOptions\
+	org.rocksdb.FlinkCompactionFilter\
 	org.rocksdb.FlushOptions\
 	org.rocksdb.Filter\
 	org.rocksdb.IngestExternalFileOptions\
@@ -75,6 +76,7 @@ NATIVE_JAVA_CLASSES = \
 	org.rocksdb.VectorMemTableConfig\
 	org.rocksdb.Snapshot\
 	org.rocksdb.StringAppendOperator\
+    org.rocksdb.StringAppendOperatorWithVariableDelimitor\
 	org.rocksdb.UInt64AddOperator\
 	org.rocksdb.WriteBatch\
 	org.rocksdb.WriteBatch.Handler\

java/crossbuild/Vagrantfile

@@ -33,7 +33,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   end
 
   config.vm.provider "virtualbox" do |v|
-    v.memory = 2048
+    v.memory = 6048
     v.cpus = 4
     v.customize ["modifyvm", :id, "--nictype1", "virtio" ]
   end

java/crossbuild/build-win.bat

@@ -0,0 +1,16 @@
+:: install git, java 8, maven, visual studio community 15 (2017)
+
+set MSBUILD=C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\MSBuild\15.0\Bin\MSBuild.exe
+
+if exist build rd /s /q build
+if exist librocksdbjni-win64.dll del librocksdbjni-win64.dll
+mkdir build && cd build
+
+cmake -G "Visual Studio 15 Win64" -DWITH_JNI=1 ..
+
+"%MSBUILD%" rocksdb.sln /p:Configuration=Release /m
+
+cd ..
+
+copy build\java\Release\rocksdbjni-shared.dll librocksdbjni-win64.dll
+echo Result is in librocksdbjni-win64.dll

java/deploysettings.xml

@@ -0,0 +1,12 @@
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
+                      http://maven.apache.org/xsd/settings-1.0.0.xsd">
+  <servers>
+    <server>
+      <id>sonatype-nexus-staging</id>
+      <username>${sonatype_user}</username>
+      <password>${sonatype_pw}</password>
+    </server>
+  </servers>
+</settings>