0.14.0

.clippy.toml

@@ -1 +1 @@
-msrv = "1.56.0"
+msrv = "1.77.0"

.codecov.yml

@@ -4,6 +4,11 @@ codecov:
 ignore:
   - integration/
   - src/map/error.rs
+  # - src/unsync/tests.rs
+  # - src/unsync/tests/
+  # - src/sync/tests.rs
+  # - src/sync/tests/
+
 
 coverage:
   status:

.github/workflows/ci.yml

@@ -106,8 +106,8 @@ jobs:
       #     cargo +nightly build --no-default-features --features alloc --target ${{ matrix.target }} -Z build-std=core,alloc
       #   if: matrix.target == 'mips64-unknown-linux-gnuabi64'
 
-  test:
-    name: test
+  build:
+    name: build
     strategy:
       matrix:
         os:
@@ -138,7 +138,39 @@ jobs:
         path: ~/.cargo
         key: ${{ runner.os }}-coverage-dotcargo
     - name: Run test
-      run: cargo hack test --feature-powerset --exclude-no-default-features --exclude-features tracing
+      run: cargo hack build --feature-powerset --exclude-no-default-features --exclude-features tracing
+
+  test:
+    name: test
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+          - macos-latest
+          - windows-latest
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v3
+    - name: Cache cargo build and registry
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.cargo/registry
+          ~/.cargo/git
+          target
+        key: ${{ runner.os }}-test-${{ hashFiles('**/Cargo.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-test-
+    - name: Install Rust
+      # --no-self-update is necessary because the windows environment cannot self-update rustup.exe.
+      run: rustup update stable --no-self-update && rustup default stable
+    - name: Cache ~/.cargo
+      uses: actions/cache@v3
+      with:
+        path: ~/.cargo
+        key: ${{ runner.os }}-coverage-dotcargo
+    - name: Run test
+      run: cargo test --all-features
 
   sanitizer:
     name: sanitizer
@@ -209,8 +241,38 @@ jobs:
         working-directory: integration
 
 
-  miri:
-    name: miri
+  miri-tb:
+    name: miri-tb
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+          - macos-latest
+          # - windows-latest
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Cache cargo build and registry
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-miri-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-miri-
+      - name: Install cargo-hack
+        run: cargo install cargo-hack
+      - name: Miri (Linux)
+        run: ci/miri_tb.sh
+        if: matrix.os == 'ubuntu-latest'
+      - name: Miri
+        run: ci/miri_tb_generic.sh
+        if: matrix.os != 'ubuntu-latest'
+
+  miri-sb:
+    name: miri-sb
     strategy:
       matrix:
         os:
@@ -233,10 +295,10 @@ jobs:
       - name: Install cargo-hack
         run: cargo install cargo-hack
       - name: Miri (Linux)
-        run: ci/miri.sh
+        run: ci/miri_sb.sh
         if: matrix.os == 'ubuntu-latest'
       - name: Miri
-        run: ci/miri_generic.sh
+        run: ci/miri_sb_generic.sh
         if: matrix.os != 'ubuntu-latest'
 
   # loom:
@@ -271,6 +333,7 @@ jobs:
     needs: 
       - rustfmt
       - clippy
+      - build
       - test
       - sanitizer
       - valgrind

CHANGELOG.md

@@ -1,5 +1,12 @@
 # CHANGELOG
 
+## 0.14.0
+
+- Supports unsync version `SkipMap`s
+- Fix: dealloc potential in-unsed memory chunk
+- Add `CompressionPolicy` as a configuration
+- Increase the discarded tracker when find new version of a key
+
 ## 0.13.0
 
 - Remove `Comparator` generic on `Entry*`
@@ -87,5 +94,3 @@
 - Add `flush` API
 
 ## UNRELEASED
-
-

Cargo.toml

@@ -1,15 +1,15 @@
 [package]
 name = "skl"
-version = "0.13.1"
+version = "0.14.0"
 edition = "2021"
-rust-version = "1.56.0"
+rust-version = "1.77.0"
 repository = "https://github.com/al8n/skl"
 description = "A lock-free thread-safe concurrent ARENA based (heap backend or memory map backend) skiplist implementation which helps develop MVCC memtable for LSM-Tree."
 documentation = "https://docs.rs/skl"
 homepage = "https://github.com/al8n/skl"
 keywords = ["skiplist", "lock-free", "memtable", "concurrency", "arena"]
 categories = ["database", "data-structures", "concurrency", "no-std"]
-license = "MIT/Apache-2.0"
+license = "MIT OR Apache-2.0"
 
 [[bench]]
 path = "benches/bench.rs"
@@ -34,7 +34,7 @@ required-features = ["memmap"]
 default = ["std"]
 alloc = ["rarena-allocator/alloc"]
 memmap = ["rarena-allocator/memmap", "std"]
-std = ["rand/default", "either/default", "rarena-allocator/std"]
+std = ["rand/default", "either/default", "rarena-allocator/std", "memchr/default", "among/default"]
 tracing = ["dep:tracing", "rarena-allocator/tracing"]
 
 # loom = ["dep:loom", "rarena-allocator/loom"]
@@ -43,11 +43,18 @@ tracing = ["dep:tracing", "rarena-allocator/tracing"]
 getrandom = { version = "0.2", features = ["js"] }
 
 [dependencies]
+among = { version = "0.1", default-features = false }
+derive_more = "0.99"
+dbutils = { version = "0.3", default-features = false }
 either = { version = "1", default-features = false }
+memchr = { version = "2", default-features = false }
 rand = { version = "0.8", default-features = false, features = ["getrandom"] }
-rarena-allocator = { version = "0.1", default-features = false }
-ux2 = { version = "0.8", default-features = false, features = ["32"] }
+# rarena-allocator = { version = "0.2", default-features = false, path = "../arena/rarena-allocator" }
+rarena-allocator = { version = "0.2", default-features = false }
+arbitrary-int = { version = "1.2", default-features = false }
+paste = "1"
 
+time = { version = "0.3", optional = true, features = ["std"] }
 tracing = { version = "0.1", optional = true }
 
 [dev-dependencies]

README.md

@@ -41,25 +41,33 @@
 ## Features
 
 - **MVCC and 3D access**: Builtin MVCC (multiple versioning concurrency control) and key-value-version access support.
-- **Lock-free and Concurrent-Safe:** SkipMap provide lock-free operations, ensuring efficient concurrent access without the need for explicit locking mechanisms.
-- **Extensible for Key-Value Database Developers:** Designed as a low-level crate, SkipMap offer a flexible foundation for key-value database developers. You can easily build your own memtable or write-ahead-log (WAL) using these structures.
+- **Lock-free and Concurrent-Safe:** `SkipMap` provide lock-free operations, ensuring efficient concurrent access without the need for explicit locking mechanisms.
+- **Extensible for Key-Value Database Developers:** Designed as a low-level crate, `SkipMap` offer a flexible foundation for key-value database developers. You can easily build your own memtable or durable storage using these structures.
 - **Memory Efficiency:** These data structures are optimized for minimal memory overhead. They operate around references, avoiding unnecessary allocations and deep copies, which can be crucial for efficient memory usage.
   - **Segment tracker:** Builtin segment recollection support, a lock-free freelist helps reuse free segments.
   - **Prefix compression:**
     - Same key will only be stored once.
     - Keys with common prefix will be stored once (longest one must be inserted first).
+    - Keys are sub-slice of negeibours will be stored once (require `CompressionPolicy::High`).
   - **Discard tracker:** Builtin discard tracker, which will track discarded bytes to help end-users decide when to compact or rewrite.
-- **Efficient Iteration:** Enjoy fast forward and backward iteration through the elements in your SkipMap. Additionally, bounded iterators are supported, allowing you to traverse only a specified range of elements efficiently.
-- **Snapshot Support:** Create snapshots of your SkipMap, offering a read-only view of the contents at a specific moment in time. Snapshots provide a consistent view of the data, enabling implementations of transactional semantics and other use cases where data consistency is crucial.
+- **Efficient Iteration:** Enjoy fast forward and backward iteration through the elements in your `SkipMap`. Additionally, bounded iterators are supported, allowing you to traverse only a specified range of elements efficiently.
+- **Snapshot Support:** Create snapshots of your `SkipMap`, offering a read-only view of the contents at a specific moment in time. Snapshots provide a consistent view of the data, enabling implementations of transactional semantics and other use cases where data consistency is crucial.
 - **Memory Management Options:**
   - **Heap Allocation:** Memory allocation is handled by Rust's allocator, ensuring all data resides in RAM.
-  - **Mmap:** Data can be mapped to a disk file by the operating system, making it suitable for write-ahead-logs (WAL) and durable storage.
+  - **Mmap:** Data can be mapped to a disk file by the operating system, making it suitable for durable storage.
   - **Mmap Anonymous:** Mapped to anonymous memory (virtual memory) by the OS, ideal for large in-memory memtables, optimizing memory utilization.
 
 ## Examples
 
 Please see [examples](https://github.com/al8n/skl/tree/main/examples) folder for more details.
 
+## Q & A
+
+- Does the on-disk version `SkipMap` ensure crash safety or power failure resilience?
+
+  No, on-disk version `SkipMap` does not ensure crash safety or power failure resilience. Hence, it is not recommended to directly
+  use the `SkipMap` as a durable database. It is recommended to use the on-disk version `SkipMap` as a final frozen file for quick lookup.
+
 ## Tests
 
 - `test`:

benches/bench.rs

@@ -1,7 +1,7 @@
 use criterion::*;
 use parking_lot::Mutex;
 use rand::prelude::*;
-use skl::*;
+use skl::{sync::map::SkipMap, *};
 use std::{
   collections::*,
   sync::{atomic::*, *},
@@ -25,11 +25,11 @@ fn fixed_map_round(
 
 fn fixed_skiplist_round(l: &SkipMap, case: &(Vec<u8>, bool), exp: &Vec<u8>) {
   if case.1 {
-    if let Some(v) = l.get(0, &case.0) {
+    if let Some(v) = l.get(&case.0) {
       assert_eq!(v.value(), exp);
     }
   } else {
-    l.insert(0, &case.0, exp).unwrap();
+    l.insert(&case.0, exp).unwrap();
   }
 }
 
@@ -42,7 +42,7 @@ fn random_key(rng: &mut ThreadRng) -> Vec<u8> {
 fn bench_read_write_fixed_skiplist_frac(b: &mut Bencher<'_>, frac: &usize) {
   let frac = *frac;
   let value = b"00123".to_vec();
-  let list = Arc::new(SkipMap::with_options(Options::new().with_capacity(512 << 20)).unwrap());
+  let list = Arc::new(SkipMap::new(Options::new().with_capacity(512 << 20)).unwrap());
   let l = list.clone();
   let stop = Arc::new(AtomicBool::new(false));
   let s = stop.clone();
@@ -164,7 +164,7 @@ fn bench_write_fixed_map(c: &mut Criterion) {
 }
 
 fn bench_write_fixed_skiplist(c: &mut Criterion) {
-  let list = Arc::new(SkipMap::with_options(Options::new().with_capacity(512 << 21)).unwrap());
+  let list = Arc::new(SkipMap::new(Options::new().with_capacity(512 << 21)).unwrap());
   let value = b"00123".to_vec();
   let l = Arc::clone(&list);
   let stop = Arc::new(AtomicBool::new(false));
@@ -182,7 +182,7 @@ fn bench_write_fixed_skiplist(c: &mut Criterion) {
     b.iter_batched(
       || random_key(&mut rng),
       |key| {
-        list.insert(0, &key, &value).unwrap();
+        list.insert(&key, &value).unwrap();
       },
       BatchSize::SmallInput,
     )

ci/miri_tb.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+rustup toolchain install nightly --component miri
+rustup override set nightly
+cargo miri setup
+
+export MIRIFLAGS="-Zmiri-strict-provenance -Zmiri-disable-isolation -Zmiri-symbolic-alignment-check -Zmiri-tree-borrows"
+
+cargo miri test --all-features --target x86_64-unknown-linux-gnu
+# cargo miri test --tests --target aarch64-unknown-linux-gnu #crossbeam_utils has problem on this platform
+cargo miri test --all-features --target i686-unknown-linux-gnu
+cargo miri test --all-features --target powerpc64-unknown-linux-gnu

ci/miri_tb_generic.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e
+
+rustup toolchain install nightly --component miri
+rustup override set nightly
+cargo miri setup
+
+export MIRIFLAGS="-Zmiri-strict-provenance -Zmiri-disable-isolation -Zmiri-symbolic-alignment-check -Zmiri-tree-borrows"
+
+cargo miri test --tests --features memmap

examples/heap.rs

@@ -1,5 +1,4 @@
-use skl::*;
-use std::sync::Arc;
+use skl::{sync::map::*, *};
 
 pub fn key(i: usize) -> Vec<u8> {
   format!("{:05}", i).into_bytes()
@@ -11,24 +10,23 @@ pub fn new_value(i: usize) -> Vec<u8> {
 
 fn main() {
   const N: usize = 1000;
-  let l = SkipMap::with_options(Options::new().with_capacity(1 << 20)).unwrap();
-  let mut wg = Arc::new(());
+
+  let l = SkipMap::new(Options::new().with_capacity(1 << 20)).unwrap();
+
   for i in 0..N {
-    let w = wg.clone();
     let l = l.clone();
     std::thread::spawn(move || {
-      l.insert(0, &key(i), &new_value(i)).unwrap();
-      drop(w);
+      l.insert(&key(i), &new_value(i)).unwrap();
     });
   }
-  while Arc::get_mut(&mut wg).is_none() {}
+
+  while l.refs() > 1 {}
+
   for i in 0..N {
-    let w = wg.clone();
     let l = l.clone();
     std::thread::spawn(move || {
       let k = key(i);
-      assert_eq!(l.get(0, &k).unwrap().value(), new_value(i), "broken: {i}");
-      drop(w);
+      assert_eq!(l.get(&k).unwrap().value(), new_value(i), "broken: {i}");
     });
   }
 }

examples/mmap.rs

@@ -1,5 +1,4 @@
-use skl::SkipMap;
-use std::sync::Arc;
+use skl::{sync::map::SkipMap, Options};
 
 pub fn key(i: usize) -> Vec<u8> {
   format!("{:05}", i).into_bytes()
@@ -17,24 +16,23 @@ fn main() {
     .read(true)
     .write(true);
 
-  let l = SkipMap::map_mut("test.wal", open_options, mmap_options).unwrap();
-  let mut wg = Arc::new(());
+  let l =
+    unsafe { SkipMap::map_mut("test.wal", Options::new(), open_options, mmap_options).unwrap() };
+
   for i in 0..N {
-    let w = wg.clone();
     let l = l.clone();
     std::thread::spawn(move || {
-      l.insert(0, &key(i), &new_value(i)).unwrap();
-      drop(w);
+      l.insert(&key(i), &new_value(i)).unwrap();
     });
   }
-  while Arc::get_mut(&mut wg).is_none() {}
+
+  while l.refs() > 1 {}
+
   for i in 0..N {
-    let w = wg.clone();
     let l = l.clone();
     std::thread::spawn(move || {
       let k = key(i);
-      assert_eq!(l.get(0, &k).unwrap().value(), new_value(i), "broken: {i}");
-      drop(w);
+      assert_eq!(l.get(&k).unwrap().value(), new_value(i), "broken: {i}");
     });
   }
 }