Address data race with KvsHeader's custom_cmp_enabled flag

Change-Id: I15d7fea7bf6dfe7409a258aa484866b7f1cf2bcc
13:09:58 WARNING: ThreadSanitizer: data race (pid=103894)
13:09:58   Read of size 1 at 0x7d100047b1a0 by thread T19:
13:09:58     #0 fdb_kvs_find_cmp_chunk /home/couchbase/jenkins/workspace/forestdb-thread_sanitizer-unstable/forestdb/src/kv_instance.cc:255 (compact_functional_test+0x0000005521bf)
13:09:58     #1 HBTrie::_insert(void*, int, void*, void*, unsigned char) /home/couchbase/jenkins/workspace/forestdb-thread_sanitizer-unstable/forestdb/src/hbtrie.cc:208 (compact_functional_test+0x000000544b6e)
13:09:58     #2 HBTrie::insert(void*, int, void*, void*) /home/couchbase/jenkins/workspace/forestdb-thread_sanitizer-unstable/forestdb/src/hbtrie.cc:1452 (compact_functional_test+0x000000544937)
13:09:58     #3 WalFlushCallbacks::flushItem(void*, wal_item*, avl_tree*, avl_tree*) /home/couchbase/jenkins/workspace/forestdb-thread_sanitizer-unstable/forestdb/src/forestdb.cc:5291 (compact_functional_test+0x00000053d016)
13:09:58     #4 Wal::_wal_do_flush(wal_item*, fdb_status (*)(void*, wal_item*, avl_tree*, avl_tree*), void*, avl_tree*, avl_tree*) /home/couchbase/jenkins/workspace/forestdb-thread_sanitizer-unstable/forestdb/src/wal.cc:1705 (compact_functional_test+0x00000056df7b)
13:09:58     #5 Wal::flushByCompactor_Wal(void*, fdb_status (*)(void*, wal_item*, avl_tree*, avl_tree*), unsigned long (*)(void*, wal_item*), void (*)(void*, avl_tree*, avl_tree*), void (*)(FileMgr*, avl_tree*), wal_flush_items*) /home/couchbase/jenkins/workspace/forestdb-thread_sanitizer-unstable/forestdb/src/wal.cc:1923 (compact_functional_test+0x00000056e321)
13:09:58     #6 Compaction::copyDocs(FdbKvsHandle*, unsigned long*, bool) /home/couchbase/jenkins/workspace/forestdb-thread_sanitizer-unstable/forestdb/src/compaction.cc:1341 (compact_functional_test+0x0000004ff2b3)
13:09:58     #7 Compaction::compactFile(FdbFileHandle*, char const*, bool, unsigned long, bool, fdb_encryption_key const*) /home/couchbase/jenkins/workspace/forestdb-thread_sanitizer-unstable/forestdb/src/compaction.cc:306 (compact_functional_test+0x0000004fc749)
13:09:58     #8 CompactionTask::run() /home/couchbase/jenkins/workspace/forestdb-thread_sanitizer-unstable/forestdb/src/compactor.cc:374 (compact_functional_test+0x00000050660f)
13:09:58     #9 ExecutorThread::run() crtstuff.c (compact_functional_test+0x00000051ac29)
13:09:58     #10 launch_executor_thread(void*) /home/couchbase/.ccache/tmp/executorth.tmp.37a3ca72313b.96623.ii (compact_functional_test+0x00000051a8d5)
13:09:58
13:09:58   Previous write of size 1 at 0x7d100047b1a0 by main thread (mutexes: write M20407327, write M20420268):
13:09:58     #0 FdbEngine::createKvs(FdbKvsHandle*, char const*, fdb_kvs_config*) /home/couchbase/jenkins/workspace/forestdb-thread_sanitizer-unstable/forestdb/src/kv_instance.cc:2129 (compact_functional_test+0x000000556a39)
13:09:58     #1 FdbEngine::openKvs(FdbKvsHandle*, fdb_config*, fdb_kvs_config*, FileMgr*, char const*, char const*, FdbKvsHandle*) /home/couchbase/jenkins/workspace/forestdb-thread_sanitizer-unstable/forestdb/src/kv_instance.cc:1803 (compact_functional_test+0x0000005562ba)
13:09:58     #2 FdbEngine::openKvs(FdbFileHandle*, FdbKvsHandle**, char const*, fdb_kvs_config*) /home/couchbase/jenkins/workspace/forestdb-thread_sanitizer-unstable/forestdb/src/kv_instance.cc:1747 (compact_functional_test+0x000000554c29)
13:09:58     #3 fdb_kvs_open /home/couchbase/jenkins/workspace/forestdb-thread_sanitizer-unstable/forestdb/src/kv_instance.cc:1065 (compact_functional_test+0x000000554780)
13:09:58     #4 auto_compaction_with_custom_cmp_function() /home/couchbase/jenkins/workspace/forestdb-thread_sanitizer-unstable/forestdb/tests/functional/compact_functional_test.cc:2125 (compact_functional_test+0x0000004d671f)
13:09:58     #5 main /home/couchbase/jenkins/workspace/forestdb-thread_sanitizer-unstable/forestdb/tests/functional/compact_functional_test.cc:3704 (compact_functional_test+0x0000004e032e)
Reviewed-on: http://review.couchbase.org/67042
Tested-by: buildbot <[email protected]>
Reviewed-by: Chiyoung Seo <[email protected]>

Author: abhinavdangeti

src/internal_types.h

@@ -387,7 +387,7 @@ class KvsHeader {
     /**
      * Boolean to determine if custom compare function for a KV store is set.
      */
-    uint8_t custom_cmp_enabled;
+    std::atomic<uint8_t> custom_cmp_enabled;
     /**
      * Number of KV store instances
      */

src/kv_instance.cc

@@ -353,8 +353,8 @@ void fdb_kvs_header_copy(FdbKvsHandle *handle,
     // copy all in-memory custom cmp function pointers & seqnums
     new_file->getKVHeader_UNLOCKED()->default_kvs_cmp =
         handle->file->getKVHeader_UNLOCKED()->default_kvs_cmp;
-    new_file->getKVHeader_UNLOCKED()->custom_cmp_enabled =
-        handle->file->getKVHeader_UNLOCKED()->custom_cmp_enabled;
+    new_file->getKVHeader_UNLOCKED()->custom_cmp_enabled.store(
+        handle->file->getKVHeader_UNLOCKED()->custom_cmp_enabled);
     a = avl_first(handle->file->getKVHeader_UNLOCKED()->idx_id);
     while (a) {
         node_old = _get_entry(a, struct kvs_node, avl_id);