From fbeea1d787096369deb7ef473bc2e2d473886b4e Mon Sep 17 00:00:00 2001
From: Thomas Leonard <talex5@gmail.com>
Date: Wed, 18 Nov 2020 15:01:44 +0000
Subject: [PATCH] Use mutex to avoid concurrent btrfs operations

Might help with problems such as this:

```
[11030132.006555] INFO: task ocluster-worker:602217 blocked for more than 120 seconds.
[11030132.015596]       Not tainted 5.4.0-40-generic #44-Ubuntu
[11030132.022547] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[11030132.032061] ocluster-worker D    0 602217      1 0x00004000
[11030132.032069] Call Trace:
[11030132.032092]  __schedule+0x2e3/0x740
[11030132.032106]  ? __switch_to_asm+0x40/0x70
[11030132.032116]  ? __switch_to_asm+0x34/0x70
[11030132.032126]  schedule+0x42/0xb0
[11030132.032130]  schedule_preempt_disabled+0xe/0x10
[11030132.032132]  __mutex_lock.isra.0+0x182/0x4f0
[11030132.032142]  ? try_to_del_timer_sync+0x54/0x80
[11030132.032145]  __mutex_lock_slowpath+0x13/0x20
[11030132.032148]  mutex_lock+0x2e/0x40
[11030132.032199]  btrfs_start_delalloc_roots+0x60/0x280 [btrfs]
[11030132.032238]  flush_space+0x5dd/0x740 [btrfs]
[11030132.032281]  ? lock_extent_buffer_for_io+0x370/0x370 [btrfs]
[11030132.032325]  ? __clear_extent_bit+0x201/0x4a0 [btrfs]
[11030132.032372]  priority_reclaim_metadata_space.isra.0+0x18b/0x220 [btrfs]
[11030132.032429]  ? can_overcommit.part.0+0x5f/0xc0 [btrfs]
[11030132.032466]  btrfs_reserve_metadata_bytes+0x578/0x950 [btrfs]
[11030132.032501]  ? btrfs_truncate_inode_items+0x35e/0xdb0 [btrfs]
[11030132.032505]  ? __mutex_lock.isra.0+0x429/0x4f0
[11030132.032557]  ? __btrfs_block_rsv_release+0x1c1/0x300 [btrfs]
[11030132.032595]  btrfs_block_rsv_refill+0x7d/0xa0 [btrfs]
[11030132.032628]  evict_refill_and_join+0x39/0xd0 [btrfs]
[11030132.032670]  btrfs_evict_inode+0x417/0x4c0 [btrfs]
[11030132.032689]  evict+0xd2/0x1b0
[11030132.032698]  iput+0x148/0x210
[11030132.032708]  dentry_unlink_inode+0xc6/0x110
[11030132.032720]  d_delete+0x76/0x80
[11030132.032727]  vfs_rmdir+0x179/0x1a0
[11030132.032732]  do_rmdir+0x18c/0x1c0
[11030132.032736]  __x64_sys_rmdir+0x17/0x20
[11030132.032744]  do_syscall_64+0x57/0x190
[11030132.032747]  entry_SYSCALL_64_after_hwframe+0x44/0xa9
```
---
 lib/btrfs_store.ml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/lib/btrfs_store.ml b/lib/btrfs_store.ml
index d90b269a..9c7cf34b 100644
--- a/lib/btrfs_store.ml
+++ b/lib/btrfs_store.ml
@@ -24,10 +24,15 @@ type t = {
 let ( / ) = Filename.concat
 
 module Btrfs = struct
+  (* Avoid doing multiple btrfs operations at once... maybe causes hangs *)
+  let lock = Lwt_mutex.create ()
+
   let btrfs ?(sudo=false) args =
     let args = "btrfs" :: args in
     let args = if sudo && not running_as_root then "sudo" :: args else args in
-    Os.exec ~stdout:`Dev_null args
+    Lwt_mutex.with_lock lock (fun () ->
+        Os.exec ~stdout:`Dev_null args
+      )
 
   let subvolume_create path =
     assert (not (Sys.file_exists path));