MB-20231: Replacing mutex with spin-lock for queue ops

+ Performance regression observed in run-time of
  fdb_extended_test.
+ Profiling showed the mutex lock contention was the
  main bottle-neck.
+ Replaced the mutex with a spin-lock
+ Run-time of fdb_extended_test back to normal.
    - Before change:
    1/1 Test #10: fdb_extended_test ................   Passed  116.43 sec
    - After change:
    1/1 Test #10: fdb_extended_test ................   Passed  103.96 sec

Change-Id: Iba5909510b99353543a8db7328e1b0ec8a35e95f
Reviewed-on: http://review.couchbase.org/66346
Reviewed-by: Chiyoung Seo <[email protected]>
Tested-by: buildbot <[email protected]>

Author: abhinavdangeti

src/memory_pool.cc

@@ -18,13 +18,15 @@
 #include <memory_pool.h>
 
 MemoryPool::MemoryPool(int num_bins, size_t bin_size) {
+    spin_init(&lock);
     for (int i = 0; i < num_bins; ++i) {
         memPool.push_back((uint8_t *) malloc(bin_size));
         enQueue(i);
     }
 }
 
 MemoryPool::~MemoryPool() {
+    spin_destroy(&lock);
     for (auto &it : memPool) {
         free(it);
     }
@@ -47,16 +49,18 @@ void MemoryPool::returnBlock(int index) {
 }
 
 inline void MemoryPool::enQueue(int index) {
-    std::lock_guard<std::mutex> guard(queueGuard);
+    spin_lock(&lock);
     indexQ.push(index);
+    spin_unlock(&lock);
 }
 
 inline int MemoryPool::deQueue() {
     int data = -1;
-    std::lock_guard<std::mutex> guard(queueGuard);
+    spin_lock(&lock);
     if (indexQ.size()) {
         data = indexQ.front();
         indexQ.pop();
     }
+    spin_unlock(&lock);
     return data;
 }

src/memory_pool.h

@@ -19,10 +19,11 @@
 
 #include <stdlib.h>
 
-#include <mutex>
 #include <queue>
 #include <vector>
 
+#include "common.h"
+
 class MemoryPool {
 /**
   The memory pool is a memory container designed for concurrent access and
@@ -81,8 +82,8 @@ class MemoryPool {
      */
     int deQueue();
 
-    // Lock to protect queue operations
-    std::mutex queueGuard;
+    // Spin lock for queue ops
+    spin_t lock;
     // Queue of indexes
     std::queue<int> indexQ;
     // Vector of pre-allocated memory