Add dynamically adjust the upper limit of the data that each layer configure option.

.gitignore

@@ -43,4 +43,7 @@ version.h
 Makefile.dep
 make_config.mk
 .vscode
-kvrocks2redis
+kvrocks2redis
+
+src/compactdb/
+src/testsdb/

kvrocks.conf

@@ -501,7 +501,7 @@ rocksdb.cache_index_and_filter_blocks yes
 # than 2 to improve performance.
 # Accept value: "no", "snappy"
 # default snappy
-rocksdb.compression snappy
+rocksdb.compression no
 
 # If non-zero, we perform bigger reads when doing compaction. If you're
 # running RocksDB on spinning disks, you should set this to at least 2MB.
@@ -597,6 +597,30 @@ rocksdb.enable_blob_garbage_collection yes
 # Default: 25
 rocksdb.blob_garbage_collection_age_cutoff 25
 
+
+# The purpose of following three options are to dynamically adjust the upper limit of
+# the data that each layer can store according to the size of the different 
+# layers of the LSM. Enabling this option will bring some improvements in
+# deletion efficiency and space amplification, but it will lose a certain 
+# amount of read performance.
+# If you want know more detail about Levels' Target Size, you can read RocksDB wiki: 
+# https://github.com/facebook/rocksdb/wiki/Leveled-Compaction#levels-target-size
+#
+# Default: no
+rocksdb.level_compaction_dynamic_level_bytes no
+
+# The total file size of level-1 sst.
+#
+# Default: 256 M
+rocksdb.max_bytes_for_level_base 268435456
+
+# Multiplication factor for the total file size of L(n+1) layers.
+# This option is a double type number in RocksDB, but kvrocks is
+# not support double data type number yet, so we use int data
+# number instead of double currently.
+#
+# Default: 10
+rocksdb.max_bytes_for_level_multiplier 10
+
 ################################ NAMESPACE #####################################
 # namespace.test change.me
-

src/config.cc

@@ -21,11 +21,15 @@ const char *kDefaultNamespace = "__namespace";
 
 const char *errNotEnableBlobDB = "Must set rocksdb.enable_blob_files to yes first.";
 
+const char *errNotSetLevelCompactionDynamicLevelBytes =
+            "Must set rocksdb.level_compaction_dynamic_level_bytes yes first.";
+
 configEnum compression_type_enum[] = {
     {"no", rocksdb::CompressionType::kNoCompression},
     {"snappy", rocksdb::CompressionType::kSnappyCompression},
     {nullptr, 0}
 };
+
 configEnum supervised_mode_enum[] = {
     {"no", SUPERVISED_NONE},
     {"auto", SUPERVISED_AUTODETECT},
@@ -145,7 +149,12 @@ Config::Config() {
       {"rocksdb.blob_file_size", false, new IntField(&RocksDB.blob_file_size, 128, 0, INT_MAX)},
       {"rocksdb.enable_blob_garbage_collection", false, new YesNoField(&RocksDB.enable_blob_garbage_collection, true)},
       {"rocksdb.blob_garbage_collection_age_cutoff",
-       false, new IntField(&RocksDB.blob_garbage_collection_age_cutoff, 25, 0, 100)}
+       false, new IntField(&RocksDB.blob_garbage_collection_age_cutoff, 25, 0, 100)},
+      {"rocksdb.max_bytes_for_level_base", false, new IntField(&RocksDB.max_bytes_for_level_base, 268435456, 0, INT16_MAX)},
+      {"rocksdb.max_bytes_for_level_multiplier",
+        false, new IntField(&RocksDB.max_bytes_for_level_multiplier, 10, 1, 100)},
+      {"rocksdb.level_compaction_dynamic_level_bytes",
+        false, new YesNoField(&RocksDB.level_compaction_dynamic_level_bytes, false)},
   };
   for (const auto &wrapper : fields) {
     auto field = wrapper.field;
@@ -404,6 +413,30 @@ void Config::initFieldCallback() {
         double cutoff = val / 100;
         return srv->storage_->SetColumnFamilyOption(trimRocksDBPrefix(k), std::to_string(cutoff));
       }},
+      {"rocksdb.max_bytes_for_level_base", [this](Server* srv, const std::string &k, const std::string& v)->Status {
+        if (!srv) return Status::OK();
+        if (!RocksDB.level_compaction_dynamic_level_bytes) {
+          return Status(Status::NotOK, errNotSetLevelCompactionDynamicLevelBytes);
+        }
+        return srv->storage_->SetColumnFamilyOption(trimRocksDBPrefix(k), v);
+      }},
+      {"rocksdb.max_bytes_for_level_multiplier", [this](Server* srv, const std::string &k,
+                                                   const std::string& v)->Status {
+        if (!srv) return Status::OK();
+        if (!RocksDB.level_compaction_dynamic_level_bytes) {
+          return Status(Status::NotOK, errNotSetLevelCompactionDynamicLevelBytes);
+        }
+        return srv->storage_->SetColumnFamilyOption(trimRocksDBPrefix(k), v);
+      }},
+      {"rocksdb.level_compaction_dynamic_level_bytes", [this](Server* srv, const std::string &k,
+                                                        const std::string& v)->Status {
+        if (!srv) return Status::OK();
+        if (!RocksDB.level_compaction_dynamic_level_bytes) {
+          return Status(Status::NotOK, errNotSetLevelCompactionDynamicLevelBytes);
+        }
+        std::string enable_blob_garbage_collection = v == "yes" ? "true" : "false";
+        return srv->storage_->SetDBOption(trimRocksDBPrefix(k), enable_blob_garbage_collection);
+      }},
       {"rocksdb.max_open_files", set_db_option_cb},
       {"rocksdb.stats_dump_period_sec", set_db_option_cb},
       {"rocksdb.delayed_write_rate", set_db_option_cb},

src/config.h

@@ -130,6 +130,9 @@ struct Config{
     int blob_file_size;
     bool enable_blob_garbage_collection;
     int blob_garbage_collection_age_cutoff;
+    int max_bytes_for_level_base;
+    int max_bytes_for_level_multiplier;
+    bool level_compaction_dynamic_level_bytes;
   } RocksDB;
 
  public:

src/storage.cc

@@ -141,6 +141,9 @@ void Storage::InitOptions(rocksdb::Options *options) {
   options->level0_slowdown_writes_trigger = config_->RocksDB.level0_slowdown_writes_trigger;
   options->level0_stop_writes_trigger = config_->RocksDB.level0_stop_writes_trigger;
   options->level0_file_num_compaction_trigger = config_->RocksDB.level0_file_num_compaction_trigger;
+  options->max_bytes_for_level_base = config_->RocksDB.max_bytes_for_level_base;
+  options->max_bytes_for_level_multiplier = config_->RocksDB.max_bytes_for_level_multiplier;
+  options->level_compaction_dynamic_level_bytes = config_->RocksDB.level_compaction_dynamic_level_bytes;
 }
 
 Status Storage::SetColumnFamilyOption(const std::string &key, const std::string &value) {

tests/config_test.cc

@@ -54,6 +54,9 @@ TEST(Config, GetAndSet) {
       {"rocksdb.blob_file_size", "128"},
       {"rocksdb.enable_blob_garbage_collection", "yes"},
       {"rocksdb.blob_garbage_collection_age_cutoff", "25"},
+      {"rocksdb.max_bytes_for_level_base", "268435456"},
+      {"rocksdb.max_bytes_for_level_multiplier", "10"},
+      {"rocksdb.level_compaction_dynamic_level_bytes", "yes"},
   };
   std::vector<std::string> values;
   for (const auto &iter : mutable_cases) {