diff --git a/Cargo.lock b/Cargo.lock index 73e4b351..22be0fea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -241,6 +241,15 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bincode" version = "2.0.0-rc.3" @@ -260,26 +269,6 @@ dependencies = [ "virtue", ] -[[package]] -name = "bindgen" -version = "0.69.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" -dependencies = [ - "bitflags 2.5.0", - "cexpr", - "clang-sys", - "itertools 0.12.1", - "lazy_static", - "lazycell", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", - "syn 2.0.72", -] - [[package]] name = "bit_field" version = "0.10.2" @@ -294,9 +283,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" dependencies = [ "serde", ] @@ -367,8 +356,6 @@ version = "1.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b62ac837cdb5cb22e10a256099b4fc502b1dfe560cb282963a974d7abd80e476" dependencies = [ - "jobserver", - "libc", "shlex", ] @@ -378,15 +365,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom", -] - [[package]] name = "cfg-if" version = "1.0.0" @@ -420,17 +398,6 @@ dependencies = [ "half", ] -[[package]] -name = "clang-sys" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" -dependencies = [ - "glob", - "libc", - "libloading", -] - [[package]] name = "clap" version = "4.5.11" @@ -683,6 +650,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.20" @@ -764,6 +740,15 @@ dependencies = [ "const-random", ] +[[package]] +name = "doxygen-rs" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "415b6ec780d34dcf624666747194393603d0373b7141eef01d12ee58881507d9" +dependencies = [ + "phf", +] + [[package]] name = "either" version = "1.13.0" @@ -851,7 +836,7 @@ dependencies = [ "anyhow", "async-trait", "base64 0.22.1", - "bincode", + "bincode 2.0.0-rc.3", "byteorder", "bzip2", "clap", @@ -870,6 +855,7 @@ dependencies = [ "flexbuffers", "futures", "hashbrown 0.14.5", + "heed", "include-flate", "indicatif", "lazy_static", @@ -879,7 +865,6 @@ dependencies = [ "num_cpus", "rand 0.9.0-alpha.1", "rayon", - "rocksdb", "serde", "serde_derive", "serde_json", @@ -900,7 +885,7 @@ name = "ferrumc_codec" version = "0.1.0" dependencies = [ "anyhow", - "bincode", + "bincode 2.0.0-rc.3", "byteorder", "deepsize", "serde", @@ -960,6 +945,15 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + [[package]] name = "futures" version = "0.3.30" @@ -1076,12 +1070,6 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" -[[package]] -name = "glob" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" - [[package]] name = "h2" version = "0.4.5" @@ -1153,6 +1141,44 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "heed" +version = "0.20.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d4f449bab7320c56003d37732a917e18798e2f1709d80263face2b4f9436ddb" +dependencies = [ + "bitflags 2.6.0", + "byteorder", + "heed-traits", + "heed-types", + "libc", + "lmdb-master-sys", + "once_cell", + "page_size", + "serde", + "synchronoise", + "url", +] + +[[package]] +name = "heed-traits" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3130048d404c57ce5a1ac61a903696e8fcde7e8c2991e9fcfc1f27c3ef74ff" + +[[package]] +name = "heed-types" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d3f528b053a6d700b2734eabcd0fd49cb8230647aa72958467527b0b7917114" +dependencies = [ + "bincode 1.3.3", + "byteorder", + "heed-traits", + "serde", + "serde_json", +] + [[package]] name = "hermit-abi" version = "0.3.3" @@ -1274,6 +1300,16 @@ dependencies = [ "tracing", ] +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "image" version = "0.23.14" @@ -1394,15 +1430,6 @@ version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" -[[package]] -name = "jobserver" -version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" -dependencies = [ - "libc", -] - [[package]] name = "js-sys" version = "0.3.69" @@ -1429,12 +1456,6 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "libc" version = "0.2.155" @@ -1465,43 +1486,6 @@ dependencies = [ "rle-decode-fast", ] -[[package]] -name = "libloading" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" -dependencies = [ - "cfg-if", - "windows-targets 0.52.5", -] - -[[package]] -name = "librocksdb-sys" -version = "0.16.0+8.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce3d60bc059831dc1c83903fb45c103f75db65c5a7bf22272764d9cc683e348c" -dependencies = [ - "bindgen", - "bzip2-sys", - "cc", - "glob", - "libc", - "libz-sys", - "lz4-sys", - "zstd-sys", -] - -[[package]] -name = "libz-sys" -version = "1.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472" -dependencies = [ - "cc", - "pkg-config", - "vcpkg", -] - [[package]] name = "linked-hash-map" version = "0.5.6" @@ -1514,6 +1498,17 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "lmdb-master-sys" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "472c3760e2a8d0f61f322fb36788021bb36d573c502b50fa3e2bcaac3ec326c9" +dependencies = [ + "cc", + "doxygen-rs", + "libc", +] + [[package]] name = "lock_api" version = "0.4.11" @@ -1530,16 +1525,6 @@ version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" -[[package]] -name = "lz4-sys" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109de74d5d2353660401699a4174a4ff23fcc649caf553df71933c7fb45ad868" -dependencies = [ - "cc", - "libc", -] - [[package]] name = "macro_rules_attribute" version = "0.2.0" @@ -1797,6 +1782,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "parking" version = "2.2.0" @@ -1889,6 +1884,48 @@ dependencies = [ "sha2", ] +[[package]] +name = "phf" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +dependencies = [ + "phf_shared", + "rand 0.8.5", +] + +[[package]] +name = "phf_macros" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "phf_shared" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.5" @@ -2109,7 +2146,7 @@ version = "11.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb9ee317cfe3fbd54b36a511efc1edd42e216903c9cd575e686dd68a2ba90d8d" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", ] [[package]] @@ -2191,16 +2228,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" -[[package]] -name = "rocksdb" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bd13e55d6d7b8cd0ea569161127567cd587676c99f4472f779a0279aa60a7a7" -dependencies = [ - "libc", - "librocksdb-sys", -] - [[package]] name = "ron" version = "0.8.1" @@ -2208,7 +2235,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" dependencies = [ "base64 0.21.7", - "bitflags 2.5.0", + "bitflags 2.6.0", "serde", "serde_derive", ] @@ -2229,12 +2256,6 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc_version" version = "0.4.0" @@ -2250,7 +2271,7 @@ version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "errno", "libc", "linux-raw-sys", @@ -2321,11 +2342,12 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.119" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8eddb61f0697cc3989c5d64b452f5488e2b8a60fd7d5076a3045076ffef8cb0" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] @@ -2417,6 +2439,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + [[package]] name = "slab" version = "0.4.9" @@ -2482,6 +2510,15 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +[[package]] +name = "synchronoise" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dbc01390fc626ce8d1cffe3376ded2b72a11bb70e1c75f404a210e4daa4def2" +dependencies = [ + "crossbeam-queue", +] + [[package]] name = "tagptr" version = "0.2.0" @@ -2547,6 +2584,21 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tinyvec" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.38.0" @@ -2803,12 +2855,27 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + [[package]] name = "unicode-ident" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "unicode-normalization" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-segmentation" version = "1.11.0" @@ -2821,6 +2888,17 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" +[[package]] +name = "url" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + [[package]] name = "utf8parse" version = "0.2.2" @@ -2854,12 +2932,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "version_check" version = "0.9.4" @@ -3224,13 +3296,3 @@ dependencies = [ "quote", "syn 2.0.72", ] - -[[package]] -name = "zstd-sys" -version = "2.0.13+zstd.1.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" -dependencies = [ - "cc", - "pkg-config", -] diff --git a/Cargo.toml b/Cargo.toml index 3dfb0a66..5dda854f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,7 +58,7 @@ nbt-lib = { path = 'src/crates/nbt-workspace/nbt-lib', features = ["derive"] } # Database moka = { version = "0.12.8", features = ["future"] } -rocksdb = { version = "0.22.0", features = ["multi-threaded-cf"] } +heed = "0.20.5" # Misc dashmap = "6.0.1" @@ -93,4 +93,4 @@ criterion = { version = "0.5.1", features = ["html_reports"] } [[bench]] name = "benches" harness = false -path = "./src/benches/bench_nbt_ser_de.rs" \ No newline at end of file +path = "./src/benches/bench_nbt_ser_de.rs" diff --git a/src/database/chunks.rs b/src/database/chunks.rs index 6daed5c3..06b3c770 100644 --- a/src/database/chunks.rs +++ b/src/database/chunks.rs @@ -1,3 +1,7 @@ +use byteorder::LE; +use heed::types::{Bytes, U64}; +use heed::Env; +use tokio::task::spawn_blocking; use tracing::{debug, warn}; use crate::database::Database; @@ -9,45 +13,104 @@ use bincode::config::standard; use bincode::{decode_from_slice, encode_to_vec}; impl Database { + + /// Fetch chunk from database + fn get_chunk_from_database(db: &Env, key: &u64) -> Result, Error> { + + // Initialize read transaction and open chunks table + let ro_tx = db.read_txn().unwrap(); + let database = db.open_database::,Bytes>(&ro_tx, Some("chunks")) + .unwrap() + .expect("No table \"chunks\" found. The database should have been initialized"); + + // Attempt to fetch chunk from table + if let Ok(data) = database.get(&ro_tx, key) { + Ok(data.map(|encoded_chunk| { + let chunk: (Chunk, usize) = decode_from_slice(encoded_chunk, standard()) + .expect("Failed to decode chunk from database"); + chunk.0 + })) + } else { + Err(Error::DatabaseError("Failed to get chunk".into())) + } + } + + /// Insert a single chunk into database + fn insert_chunk_into_database(db: &Env, chunk: &Chunk) -> Result<(), Error> { + + // Initialize write transaction and open chunks table + let mut rw_tx = db.write_txn().unwrap(); + let database = db.open_database::,Bytes>(&rw_tx, Some("chunks")) + .unwrap() + .expect("No table \"chunks\" found. The database should have been initialized"); + + // Encode chunk + let encoded_chunk = encode_to_vec(chunk, standard()).expect("Failed to encode chunk"); + let key = hash((chunk.dimension.as_ref().unwrap(), chunk.x_pos, chunk.z_pos)); + + // Insert chunk + let res = database.put(&mut rw_tx, &key, &encoded_chunk); + rw_tx.commit() + .map_err(|err| Error::DatabaseError(format!("Unable to commit changes to database: {err}")))?; + + if let Err(err) = res { + Err(Error::DatabaseError(format!("Failed to insert or update chunk: {err}"))) + } else { + Ok(()) + } + } + + /// Insert multiple chunks into database + /// TODO: Find better name/disambiguation + fn insert_chunks_into_database(db: &Env, chunks: &[Chunk]) -> Result<(), Error> { + + // Initialize write transaction and open chunks table + let mut rw_tx = db.write_txn().unwrap(); + let database = db.open_database::,Bytes>(&rw_tx, Some("chunks")) + .unwrap() + .expect("No table \"chunks\" found. The database should have been initialized"); + + // Update page + for chunk in chunks { + // Encode chunk + let encoded_chunk = encode_to_vec(chunk, standard()).expect("Failed to encode chunk"); + let key = hash((chunk.dimension.as_ref().unwrap(), chunk.x_pos, chunk.z_pos)); + + // Insert chunk + database.put(&mut rw_tx, &key, &encoded_chunk).map_err(|err| Error::DatabaseError(format!("Failed to insert or update chunk: {err}")))?; + } + + // Commit changes + rw_tx.commit() + .map_err(|err| Error::DatabaseError(format!("Unable to commit changes to database: {err}")))?; + Ok(()) + } + async fn load_into_cache(&self, key: u64) -> Result<(), Error> { let db = self.db.clone(); let cache = self.cache.clone(); tokio::task::spawn(async move { - // This is stupid, but it's the only way to get the lifetime checker to shut up - let get_chunk = |db: &rocksdb::DB, key: u64| { - let cf = db - .cf_handle("chunks") - .expect("Failed to get column family \"chunks\""); - if let Ok(data) = db.get_cf(&cf, key.to_be_bytes()) { - if let Some(encoded) = data { - let chunk: (Chunk, usize) = decode_from_slice(&encoded, standard()) - .expect("Failed to decode chunk from database"); - Ok(Some(chunk.0)) - } else { - Ok(None) - } - } else { - Err(Error::DatabaseError("Failed to get chunk".to_string())) - } - }; + // Check cache if cache.contains_key(&key) { debug!("Chunk already exists in cache: {:X}", key); - } else { - if let Ok(c) = get_chunk(&db, key) { - if c.is_some() { - cache.insert(key, c.unwrap()).await; - } else { - warn!( - "Chunk does not exist in db, can't load into cache: {:X}", - key, - ); - } + } + // If not in cache then search in database + else if let Ok(chunk) = spawn_blocking(move || Self::get_chunk_from_database(&db, &key)).await.unwrap() { + if let Some(chunk) = chunk { + cache.insert(key, chunk).await; } else { - warn!("Error getting chunk: {:X}", key,); + warn!( + "Chunk does not exist in db, can't load into cache: {:X}", + key, + ); } } + // The chunk don't exist + else { + warn!("Error getting chunk: {:X}", key,); + } }) .await?; Ok(()) @@ -72,29 +135,18 @@ impl Database { /// /// ``` pub async fn insert_chunk(&self, value: Chunk) -> Result<(), Error> { - let key = hash((value.x_pos, value.z_pos)); - let result = self.internal_insert_chunk(value.clone()).await; - if result.is_ok() { - self.cache.insert(key, value).await; - Ok(()) - } else { - result - } - } - async fn internal_insert_chunk(&self, value: Chunk) -> Result<(), Error> { + // Calculate key of this chunk + // WARNING: This key wasn't supposed to include value.dimension in the tuple but it was different from the key used in persistent database most likely a bug. + let key = hash((value.dimension.as_ref().unwrap(), value.x_pos, value.z_pos)); + + // Insert chunk into persistent database + let chunk = value.clone(); let db = self.db.clone(); - tokio::task::spawn_blocking(move || { - let cf = db - .cf_handle("chunks") - .expect("Failed to get column family \"chunks\""); - let encoded = encode_to_vec(&value, standard()).expect("Failed to encode chunk"); - let key = hash((value.dimension.unwrap(), value.x_pos, value.z_pos)); - db.put_cf(&cf, key.to_be_bytes(), encoded) - .or(Err(Error::DatabaseError( - "Failed to insert chunk".to_string(), - ))) - }) - .await? + spawn_blocking(move || Self::insert_chunk_into_database(&db, &chunk)).await.unwrap()?; + + // Insert into cache + self.cache.insert(key, value).await; + Ok(()) } /// Get a chunk from the database
@@ -123,40 +175,23 @@ impl Database { z: i32, dimension: String, ) -> Result, Error> { + // Calculate key of this chunk and clone database pointer let key = hash((dimension, x, z)); + let db = self.db.clone(); + + // First check cache if self.cache.contains_key(&key) { Ok(self.cache.get(&key).await) - } else { - if let Ok(chunk) = self.internal_get_chunk(key).await { - if chunk.is_some() { - self.cache.insert(key, chunk.clone().unwrap()).await; - } - Ok(chunk) - } else { - Ok(None) - } } - } - - async fn internal_get_chunk(&self, key: u64) -> Result, Error> { - let db = self.db.clone(); - tokio::task::spawn_blocking(move || { - let cf = db - .cf_handle("chunks") - .expect("Failed to get column family \"chunks\""); - if let Ok(data) = db.get_cf(&cf, key.to_be_bytes()) { - if let Some(encoded) = data { - let chunk = decode_from_slice(&encoded, standard()) - .expect("Failed to decode chunk from database"); - Ok(Some(chunk.0)) - } else { - Ok(None) - } - } else { - Err(Error::DatabaseError("Failed to get chunk".to_string())) - } - }) - .await? + // Attempt to get chunk from persistent database + else if let Some(chunk) = spawn_blocking(move || Self::get_chunk_from_database(&db, &key)).await.unwrap()? { + self.cache.insert(key, chunk.clone()).await; + Ok(Some(chunk)) + } + // Chunk do not exist + else { + Ok(None) + } } /// Check if a chunk exists in the database @@ -178,24 +213,32 @@ impl Database { /// /// ``` pub async fn chunk_exists(&self, x: i32, z: i32, dimension: String) -> Result { + // Calculate key and copy database pointer let key = hash((dimension, x, z)); + let db = self.db.clone(); + + // Check first cache if self.cache.contains_key(&key) { Ok(true) + // Else check persistent database and load it into cache } else { - let res = self.internal_chunk_exists(key).await; - self.load_into_cache(key).await?; - res + let res = spawn_blocking(move || Self::get_chunk_from_database(&db, &key)).await.unwrap(); + + // WARNING: The previous logic was to order the chunk to be loaded into cache whether it existed or not. + // This has been replaced by directly loading the queried chunk into cache + match res { + Ok(opt) => { + let exist = opt.is_some(); + if let Some(chunk) = opt { + self.cache.insert(key, chunk).await; + } + Ok(exist) + } + Err(err) => Err(err) + } } } - async fn internal_chunk_exists(&self, key: u64) -> Result { - let cf = self - .db - .cf_handle("chunks") - .expect("Failed to get column family \"chunks\""); - Ok(self.db.get_cf(&cf, key.to_be_bytes()).is_ok()) - } - /// Update a chunk in the database
/// This will also update the chunk in the cache
/// If the chunk does not exist, it will return an error @@ -215,31 +258,18 @@ impl Database { /// /// ``` pub async fn update_chunk(&self, value: Chunk) -> Result<(), Error> { - let key = hash((value.x_pos, value.z_pos)); - let result = self.internal_update_chunk(value.clone()).await; - if result.is_ok() { - self.cache.insert(key, value).await; - Ok(()) - } else { - result - } - } - - async fn internal_update_chunk(&self, value: Chunk) -> Result<(), Error> { + // Calculate key of this chunk + // WARNING: This key wasn't supposed to include value.dimension in the tuple but it was different from the key used in persistent database most likely a bug. + let key = hash((value.dimension.as_ref().unwrap(), value.x_pos, value.z_pos)); + + // Insert new chunk state into persistent database + let chunk = value.clone(); let db = self.db.clone(); - - tokio::task::spawn_blocking(move || { - let cf = db - .cf_handle("chunks") - .expect("Failed to get column family \"chunks\""); - let encoded = encode_to_vec(&value, standard()).expect("Failed to encode chunk"); - let key = hash((value.dimension.unwrap(), value.x_pos, value.z_pos)); - db.put_cf(&cf, key.to_be_bytes(), encoded) - .or(Err(Error::DatabaseError( - "Failed to update chunk".to_string(), - ))) - }) - .await? + spawn_blocking(move || Self::insert_chunk_into_database(&db, &chunk)).await.unwrap()?; + + // Insert new chunk state into cache + self.cache.insert(key, value).await; + Ok(()) } /// Batch insert chunks into the database
@@ -261,40 +291,26 @@ impl Database { /// /// ``` pub async fn batch_insert(&self, values: Vec) -> Result<(), Error> { - // TODO: Ewwwww clones, disgusting, fix this + // Clone database pointer + let db = self.db.clone(); + + // Calculate all keys let keys = values .iter() .map(|v| hash((v.dimension.as_ref().unwrap(), v.x_pos, v.z_pos))) .collect::>(); - let result = self.internal_batch_insert(values).await; - if result.is_ok() { - for key in keys { - self.load_into_cache(key).await?; - } - Ok(()) - } else { - result + + // WARNING: The previous logic was to first insert in database and then insert in cache using load_into_cache fn. + // This has been modified to avoid having to query database while we already have the data available. + // First insert into cache + for (key, chunk) in keys.into_iter().zip(&values) { + self.cache.insert(key, chunk.clone()).await; + self.load_into_cache(key).await?; } - } - - async fn internal_batch_insert(&self, values: Vec) -> Result<(), Error> { - let db = self.db.clone(); - - tokio::task::spawn_blocking(move || { - let cf = db - .cf_handle("chunks") - .expect("Failed to get column family \"chunks\""); - let mut batch = rocksdb::WriteBatch::default(); - for value in values { - let encoded = encode_to_vec(&value, standard()).expect("Failed to encode chunk"); - let key = hash((value.dimension.unwrap(), value.x_pos, value.z_pos)); - batch.put_cf(&cf, key.to_be_bytes(), encoded); - } - db.write(batch).or(Err(Error::DatabaseError( - "Failed to batch insert chunks".to_string(), - ))) - }) - .await? + + // Then insert into persistent database + spawn_blocking(move || Self::insert_chunks_into_database(&db, &values)).await.unwrap()?; + Ok(()) } } diff --git a/src/database/mod.rs b/src/database/mod.rs index 244ba1a6..bdf2e286 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -1,7 +1,9 @@ +use byteorder::LE; use deepsize::DeepSizeOf; use futures::FutureExt; +use heed::types::{Bytes, U64}; +use heed::{Env as LMDBDatabase, EnvFlags, EnvOpenOptions}; use moka::notification::{ListenerFuture, RemovalCause}; -use rocksdb::{Cache, ColumnFamilyDescriptor, Options, DB}; use std::env; use std::path::PathBuf; use std::sync::Arc; @@ -13,11 +15,19 @@ use crate::utils::config::get_global_config; use crate::utils::error::Error; use crate::world::chunkformat::Chunk; - pub mod chunks; + +// MDBX constants +const LMDB_MAX_TABLE: u32 = 16; +const LMDB_PAGE_SIZE: usize = 50*1024usize.pow(3); // 50GiB + +/// Global database structure +/// +/// Internally contain a handle to the persistent database and a +/// cache for all in-memory updates pub struct Database { - db: Arc, + db: LMDBDatabase, cache: Arc>, } @@ -30,7 +40,10 @@ fn evict_chunk(_key: Arc, value: Chunk, cause: RemovalCause) -> ListenerFut .boxed() } +/// Start database pub async fn start_database() -> Result { + + // Parse root directory from environment variable let root = if env::var("FERRUMC_ROOT").is_ok() { PathBuf::from(env::var("FERRUMC_ROOT").unwrap()) } else { @@ -42,6 +55,7 @@ pub async fn start_database() -> Result { ) }; + // Obtain global config to locate which world folder to load let world = get_global_config().world.clone(); let world_path = root.join("data").join(world); @@ -51,42 +65,30 @@ pub async fn start_database() -> Result { fs::create_dir_all(&world_path).await?; } - let mut opts = Options::default(); - opts.create_if_missing(true); - opts.create_missing_column_families(true); - opts.increase_parallelism(num_cpus::get() as i32); - opts.set_db_log_dir(root.join("logs")); - opts.set_compression_type(rocksdb::DBCompressionType::Lz4); - // opts.set_compression_options_parallel_threads(num_cpus::get() as i32); - let cache = Cache::new_lru_cache(512 * 1024); // 1MB cache - { - let mut bb_opts = rocksdb::BlockBasedOptions::default(); - bb_opts.set_block_cache(&cache); - bb_opts.set_checksum_type(rocksdb::ChecksumType::NoChecksum); - opts.set_block_based_table_factory(&bb_opts); + // Database Options + let mut opts = EnvOpenOptions::new(); + opts + .max_readers(num_cpus::get() as u32) + .map_size(LMDB_PAGE_SIZE) + .max_dbs(LMDB_MAX_TABLE); + + // Open database (This operation is safe as we assume no other process touched the database) + let lmdb = unsafe { opts.flags(EnvFlags::WRITE_MAP).open(&world_path).expect("Unable to open LMDB environment located at {world_path:?}") }; + + // Check if database is built. Otherwise initialize it + let mut rw_tx = lmdb.write_txn().unwrap(); + if lmdb.open_database::, Bytes>(&rw_tx, Some("chunks")).unwrap().is_none() { + lmdb.create_database::, Bytes>(&mut rw_tx, Some("chunks")).expect("Unable to create database"); } - opts.set_row_cache(&cache); - opts.set_paranoid_checks(false); - opts.set_disable_auto_compactions(true); - opts.set_compaction_readahead_size(0); - opts.set_allow_mmap_reads(true); - opts.set_allow_mmap_writes(true); - - let cf_names = vec!["chunks", "entities"]; - let cf_descriptors = cf_names - .into_iter() - .map(|name| { - ColumnFamilyDescriptor::new(name, opts.clone()) - }) - .collect::>(); - - let database = DB::open_cf_descriptors(&opts, world_path, cf_descriptors) - .expect("Failed to open database"); - + // `entities` table to be added, but needs the type to do so + + rw_tx.commit().unwrap(); + info!("Database started"); info!("Initializing cache"); + // Initializing moka cache let cache = moka::future::Cache::builder() .async_eviction_listener(evict_chunk) .weigher(|_, v| v.deep_size_of() as u32) @@ -96,8 +98,7 @@ pub async fn start_database() -> Result { .build(); Ok(Database { - db: Arc::new(database), + db: lmdb, cache: Arc::new(cache), }) } -