diff --git a/db/column_family_test.cc b/db/column_family_test.cc index 5c2a068b34d..e8a0420f3c8 100644 --- a/db/column_family_test.cc +++ b/db/column_family_test.cc @@ -513,6 +513,16 @@ class ColumnFamilyTestBase : public testing::Test { return static_cast(files.size()); } + void RecalculateWriteStallConditions(ColumnFamilyData* cfd, + const MutableCFOptions& mutable_cf_options) { + // add lock to avoid race condition between + // `RecalculateWriteStallConditions` which writes to CFStats and + // background `DBImpl::DumpStats()` threads which read CFStats + dbfull()->TEST_LockMutex(); + cfd->RecalculateWriteStallConditions(mutable_cf_options); + dbfull()-> TEST_UnlockMutex(); + } + std::vector handles_; std::vector names_; std::vector> keys_; @@ -2500,139 +2510,139 @@ TEST_P(ColumnFamilyTest, WriteStallSingleColumnFamily) { mutable_cf_options.disable_auto_compactions = false; vstorage->TEST_set_estimated_compaction_needed_bytes(50); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); vstorage->TEST_set_estimated_compaction_needed_bytes(201); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate()); ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage->TEST_set_estimated_compaction_needed_bytes(400); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate()); ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage->TEST_set_estimated_compaction_needed_bytes(500); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate / 1.25 / 1.25, GetDbDelayedWriteRate()); vstorage->TEST_set_estimated_compaction_needed_bytes(450); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate()); vstorage->TEST_set_estimated_compaction_needed_bytes(205); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate()); vstorage->TEST_set_estimated_compaction_needed_bytes(202); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate()); vstorage->TEST_set_estimated_compaction_needed_bytes(201); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate()); vstorage->TEST_set_estimated_compaction_needed_bytes(198); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); vstorage->TEST_set_estimated_compaction_needed_bytes(399); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate()); vstorage->TEST_set_estimated_compaction_needed_bytes(599); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate()); vstorage->TEST_set_estimated_compaction_needed_bytes(2001); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(IsDbWriteStopped()); ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage->TEST_set_estimated_compaction_needed_bytes(3001); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(IsDbWriteStopped()); ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); vstorage->TEST_set_estimated_compaction_needed_bytes(390); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate()); vstorage->TEST_set_estimated_compaction_needed_bytes(100); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); vstorage->set_l0_delay_trigger_count(100); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate()); ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage->set_l0_delay_trigger_count(101); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate()); vstorage->set_l0_delay_trigger_count(0); vstorage->TEST_set_estimated_compaction_needed_bytes(300); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate / 1.25 / 1.25, GetDbDelayedWriteRate()); vstorage->set_l0_delay_trigger_count(101); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate / 1.25 / 1.25 / 1.25, GetDbDelayedWriteRate()); vstorage->TEST_set_estimated_compaction_needed_bytes(200); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate / 1.25 / 1.25, GetDbDelayedWriteRate()); vstorage->set_l0_delay_trigger_count(0); vstorage->TEST_set_estimated_compaction_needed_bytes(0); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); mutable_cf_options.disable_auto_compactions = true; dbfull()->TEST_write_controler().set_delayed_write_rate(kBaseRate); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); vstorage->set_l0_delay_trigger_count(50); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(0, GetDbDelayedWriteRate()); @@ -2640,7 +2650,7 @@ TEST_P(ColumnFamilyTest, WriteStallSingleColumnFamily) { vstorage->set_l0_delay_trigger_count(60); vstorage->TEST_set_estimated_compaction_needed_bytes(300); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(0, GetDbDelayedWriteRate()); @@ -2649,14 +2659,14 @@ TEST_P(ColumnFamilyTest, WriteStallSingleColumnFamily) { mutable_cf_options.disable_auto_compactions = false; vstorage->set_l0_delay_trigger_count(70); vstorage->TEST_set_estimated_compaction_needed_bytes(500); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate()); vstorage->set_l0_delay_trigger_count(71); vstorage->TEST_set_estimated_compaction_needed_bytes(501); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate()); @@ -2681,31 +2691,31 @@ TEST_P(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) { mutable_cf_options.hard_pending_compaction_bytes_limit = 2000; vstorage->TEST_set_estimated_compaction_needed_bytes(40); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); vstorage->TEST_set_estimated_compaction_needed_bytes(50); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage->TEST_set_estimated_compaction_needed_bytes(300); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage->TEST_set_estimated_compaction_needed_bytes(45); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); vstorage->set_l0_delay_trigger_count(7); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); vstorage->set_l0_delay_trigger_count(9); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage->set_l0_delay_trigger_count(6); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); // Speed up threshold = min(4 * 2, 4 + (12 - 4)/4) = 6 @@ -2714,15 +2724,15 @@ TEST_P(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) { mutable_cf_options.level0_stop_writes_trigger = 30; vstorage->set_l0_delay_trigger_count(5); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); vstorage->set_l0_delay_trigger_count(7); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage->set_l0_delay_trigger_count(3); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); } @@ -2749,53 +2759,53 @@ TEST_P(ColumnFamilyTest, WriteStallTwoColumnFamilies) { mutable_cf_options1.soft_pending_compaction_bytes_limit = 500; vstorage->TEST_set_estimated_compaction_needed_bytes(50); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); vstorage1->TEST_set_estimated_compaction_needed_bytes(201); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay()); vstorage1->TEST_set_estimated_compaction_needed_bytes(600); - cfd1->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd1, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate()); vstorage->TEST_set_estimated_compaction_needed_bytes(70); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate()); vstorage1->TEST_set_estimated_compaction_needed_bytes(800); - cfd1->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd1, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate()); vstorage->TEST_set_estimated_compaction_needed_bytes(300); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate / 1.25 / 1.25, GetDbDelayedWriteRate()); vstorage1->TEST_set_estimated_compaction_needed_bytes(700); - cfd1->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd1, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate()); vstorage->TEST_set_estimated_compaction_needed_bytes(500); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate / 1.25 / 1.25, GetDbDelayedWriteRate()); vstorage1->TEST_set_estimated_compaction_needed_bytes(600); - cfd1->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd1, mutable_cf_options); ASSERT_TRUE(!IsDbWriteStopped()); ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay()); ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate()); @@ -2828,41 +2838,41 @@ TEST_P(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) { mutable_cf_options1.level0_slowdown_writes_trigger = 16; vstorage->TEST_set_estimated_compaction_needed_bytes(40); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); vstorage->TEST_set_estimated_compaction_needed_bytes(60); - cfd1->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd1, mutable_cf_options); ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage1->TEST_set_estimated_compaction_needed_bytes(30); - cfd1->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd1, mutable_cf_options); ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage1->TEST_set_estimated_compaction_needed_bytes(70); - cfd1->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd1, mutable_cf_options); ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage->TEST_set_estimated_compaction_needed_bytes(20); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage1->TEST_set_estimated_compaction_needed_bytes(3); - cfd1->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd1, mutable_cf_options); ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); vstorage->set_l0_delay_trigger_count(9); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage1->set_l0_delay_trigger_count(2); - cfd1->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd1, mutable_cf_options); ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed()); vstorage->set_l0_delay_trigger_count(0); - cfd->RecalculateWriteStallConditions(mutable_cf_options); + RecalculateWriteStallConditions(cfd, mutable_cf_options); ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed()); }