fix: deadlock (#43)

* fix: deadlock fix deadlocks: 1. exclusive lock was not `Send` but unsafe impled. 2. submit flush task await blocks `set_region_evictable` 3. no new reclamation task created after `set_region_evctable` if all write process are waiting Signed-off-by: MrCroxx <[email protected]> * update ci Signed-off-by: MrCroxx <[email protected]> --------- Signed-off-by: MrCroxx <[email protected]>
foyer-rs · Jul 4, 2023 · f067d1c · f067d1c
1 parent 691054a
commit f067d1c
Show file tree

Hide file tree

Showing 14 changed files with 1,019 additions and 386 deletions.
diff --git a/.github/template/template.yml b/.github/template/template.yml
@@ -70,6 +70,37 @@ jobs:
         run: |
           cargo llvm-cov nextest --lcov --output-path lcov.info
       - uses: codecov/codecov-action@v2
+  deadlock:
+    name: run with single worker thread and deadlock detection
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install rust toolchain@v1
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          # components: rustfmt, clippy, llvm-tools-preview
+      - name: Cache Cargo home
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.lock') }}-${{ env.CACHE_KEY_SUFFIX }}
+      - name: Run foyer-storage-bench with single worker thread and deadlock detection
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: '--cfg tokio_unstable'
+          RUST_LOG: info
+          TOKIO_WORKER_THREADS: 1
+        run: |-
+          cargo build --all --features deadlock &&
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock &&
+          timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --time 60
   asan:
     name: run with address saniziter
     runs-on: ubuntu-latest
@@ -95,8 +126,8 @@ jobs:
         env:
           RUST_BACKTRACE: 1
           RUSTFLAGS: '-Zsanitizer=address --cfg tokio_unstable'
-          EXTRA_CARGO_ARGS: '--verbose -Zbuild-std --target x86_64-unknown-linux-gnu'
+          RUST_LOG: info
         run: |-
           cargo build --all --target x86_64-unknown-linux-gnu &&
-          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench &&
-          ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan &&
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --time 60
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -77,6 +77,36 @@ jobs:
         run: |
           cargo llvm-cov nextest --lcov --output-path lcov.info
       - uses: codecov/codecov-action@v2
+  deadlock:
+    name: run with single worker thread and deadlock detection
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install rust toolchain@v1
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - name: Cache Cargo home
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.lock') }}-${{ env.CACHE_KEY_SUFFIX }}
+      - name: Run foyer-storage-bench with single worker thread and deadlock detection
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: '--cfg tokio_unstable'
+          RUST_LOG: info
+          TOKIO_WORKER_THREADS: 1
+        run: |-
+          cargo build --all --features deadlock &&
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock &&
+          timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --time 60
   asan:
     name: run with address saniziter
     runs-on: ubuntu-latest
@@ -101,11 +131,11 @@ jobs:
         env:
           RUST_BACKTRACE: 1
           RUSTFLAGS: '-Zsanitizer=address --cfg tokio_unstable'
-          EXTRA_CARGO_ARGS: '--verbose -Zbuild-std --target x86_64-unknown-linux-gnu'
+          RUST_LOG: info
         run: |-
           cargo build --all --target x86_64-unknown-linux-gnu &&
-          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench &&
-          ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan &&
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --time 60
 
 # ================= THIS FILE IS AUTOMATICALLY GENERATED =================
 #

diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
@@ -76,6 +76,36 @@ jobs:
         run: |
           cargo llvm-cov nextest --lcov --output-path lcov.info
       - uses: codecov/codecov-action@v2
+  deadlock:
+    name: run with single worker thread and deadlock detection
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install rust toolchain@v1
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - name: Cache Cargo home
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ runner.os }}-cargo-asan-${{ hashFiles('**/Cargo.lock') }}-${{ env.CACHE_KEY_SUFFIX }}
+      - name: Run foyer-storage-bench with single worker thread and deadlock detection
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: '--cfg tokio_unstable'
+          RUST_LOG: info
+          TOKIO_WORKER_THREADS: 1
+        run: |-
+          cargo build --all --features deadlock &&
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock &&
+          timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --time 60
   asan:
     name: run with address saniziter
     runs-on: ubuntu-latest
@@ -100,11 +130,11 @@ jobs:
         env:
           RUST_BACKTRACE: 1
           RUSTFLAGS: '-Zsanitizer=address --cfg tokio_unstable'
-          EXTRA_CARGO_ARGS: '--verbose -Zbuild-std --target x86_64-unknown-linux-gnu'
+          RUST_LOG: info
         run: |-
           cargo build --all --target x86_64-unknown-linux-gnu &&
-          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench &&
-          ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan &&
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --buffer-pool-size 256 --lookup-range 1000 --time 60
 concurrency:
   group: environment-${{ github.ref }}
   cancel-in-progress: true