Skip to content

Commit 936b461

Browse files
sipasipa
committed
Merge upstream LevelDB 1.13.
2 parents be1b0ff + 748539c commit 936b461

13 files changed

+363
-67
lines changed

‎Makefile

+5-1
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ TESTHARNESS = ./util/testharness.o $(TESTUTIL)
3131

3232
TESTS = \
3333
arena_test \
34+
autocompact_test \
3435
bloom_test \
3536
c_test \
3637
cache_test \
@@ -70,7 +71,7 @@ SHARED = $(SHARED1)
7071
else
7172
# Update db.h if you change these.
7273
SHARED_MAJOR = 1
73-
SHARED_MINOR = 12
74+
SHARED_MINOR = 13
7475
SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT)
7576
SHARED2 = $(SHARED1).$(SHARED_MAJOR)
7677
SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR)
@@ -114,6 +115,9 @@ leveldbutil: db/leveldb_main.o $(LIBOBJECTS)
114115
arena_test: util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS)
115116
$(CXX) $(LDFLAGS) util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
116117

118+
autocompact_test: db/autocompact_test.o $(LIBOBJECTS) $(TESTHARNESS)
119+
$(CXX) $(LDFLAGS) db/autocompact_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
120+
117121
bloom_test: util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS)
118122
$(CXX) $(LDFLAGS) util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
119123

‎db/autocompact_test.cc

+118
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
// Copyright (c) 2013 The LevelDB Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file. See the AUTHORS file for names of contributors.
4+
5+
#include "leveldb/db.h"
6+
#include "db/db_impl.h"
7+
#include "leveldb/cache.h"
8+
#include "util/testharness.h"
9+
#include "util/testutil.h"
10+
11+
namespace leveldb {
12+
13+
class AutoCompactTest {
14+
public:
15+
std::string dbname_;
16+
Cache* tiny_cache_;
17+
Options options_;
18+
DB* db_;
19+
20+
AutoCompactTest() {
21+
dbname_ = test::TmpDir() + "/autocompact_test";
22+
tiny_cache_ = NewLRUCache(100);
23+
options_.block_cache = tiny_cache_;
24+
DestroyDB(dbname_, options_);
25+
options_.create_if_missing = true;
26+
options_.compression = kNoCompression;
27+
ASSERT_OK(DB::Open(options_, dbname_, &db_));
28+
}
29+
30+
~AutoCompactTest() {
31+
delete db_;
32+
DestroyDB(dbname_, Options());
33+
delete tiny_cache_;
34+
}
35+
36+
std::string Key(int i) {
37+
char buf[100];
38+
snprintf(buf, sizeof(buf), "key%06d", i);
39+
return std::string(buf);
40+
}
41+
42+
uint64_t Size(const Slice& start, const Slice& limit) {
43+
Range r(start, limit);
44+
uint64_t size;
45+
db_->GetApproximateSizes(&r, 1, &size);
46+
return size;
47+
}
48+
49+
void DoReads(int n);
50+
};
51+
52+
static const int kValueSize = 200 * 1024;
53+
static const int kTotalSize = 100 * 1024 * 1024;
54+
static const int kCount = kTotalSize / kValueSize;
55+
56+
// Read through the first n keys repeatedly and check that they get
57+
// compacted (verified by checking the size of the key space).
58+
void AutoCompactTest::DoReads(int n) {
59+
std::string value(kValueSize, 'x');
60+
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
61+
62+
// Fill database
63+
for (int i = 0; i < kCount; i++) {
64+
ASSERT_OK(db_->Put(WriteOptions(), Key(i), value));
65+
}
66+
ASSERT_OK(dbi->TEST_CompactMemTable());
67+
68+
// Delete everything
69+
for (int i = 0; i < kCount; i++) {
70+
ASSERT_OK(db_->Delete(WriteOptions(), Key(i)));
71+
}
72+
ASSERT_OK(dbi->TEST_CompactMemTable());
73+
74+
// Get initial measurement of the space we will be reading.
75+
const int64_t initial_size = Size(Key(0), Key(n));
76+
const int64_t initial_other_size = Size(Key(n), Key(kCount));
77+
78+
// Read until size drops significantly.
79+
std::string limit_key = Key(n);
80+
for (int read = 0; true; read++) {
81+
ASSERT_LT(read, 100) << "Taking too long to compact";
82+
Iterator* iter = db_->NewIterator(ReadOptions());
83+
for (iter->SeekToFirst();
84+
iter->Valid() && iter->key().ToString() < limit_key;
85+
iter->Next()) {
86+
// Drop data
87+
}
88+
delete iter;
89+
// Wait a little bit to allow any triggered compactions to complete.
90+
Env::Default()->SleepForMicroseconds(1000000);
91+
uint64_t size = Size(Key(0), Key(n));
92+
fprintf(stderr, "iter %3d => %7.3f MB [other %7.3f MB]\n",
93+
read+1, size/1048576.0, Size(Key(n), Key(kCount))/1048576.0);
94+
if (size <= initial_size/10) {
95+
break;
96+
}
97+
}
98+
99+
// Verify that the size of the key space not touched by the reads
100+
// is pretty much unchanged.
101+
const int64_t final_other_size = Size(Key(n), Key(kCount));
102+
ASSERT_LE(final_other_size, initial_other_size + 1048576);
103+
ASSERT_GE(final_other_size, initial_other_size/5 - 1048576);
104+
}
105+
106+
TEST(AutoCompactTest, ReadAll) {
107+
DoReads(kCount);
108+
}
109+
110+
TEST(AutoCompactTest, ReadHalf) {
111+
DoReads(kCount/2);
112+
}
113+
114+
} // namespace leveldb
115+
116+
int main(int argc, char** argv) {
117+
return leveldb::test::RunAllTests();
118+
}

‎db/corruption_test.cc

+22-29
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ class CorruptionTest {
3535
CorruptionTest() {
3636
tiny_cache_ = NewLRUCache(100);
3737
options_.env = &env_;
38+
options_.block_cache = tiny_cache_;
3839
dbname_ = test::TmpDir() + "/db_test";
3940
DestroyDB(dbname_, options_);
4041

@@ -50,17 +51,14 @@ class CorruptionTest {
5051
delete tiny_cache_;
5152
}
5253

53-
Status TryReopen(Options* options = NULL) {
54+
Status TryReopen() {
5455
delete db_;
5556
db_ = NULL;
56-
Options opt = (options ? *options : options_);
57-
opt.env = &env_;
58-
opt.block_cache = tiny_cache_;
59-
return DB::Open(opt, dbname_, &db_);
57+
return DB::Open(options_, dbname_, &db_);
6058
}
6159

62-
void Reopen(Options* options = NULL) {
63-
ASSERT_OK(TryReopen(options));
60+
void Reopen() {
61+
ASSERT_OK(TryReopen());
6462
}
6563

6664
void RepairDB() {
@@ -92,6 +90,10 @@ class CorruptionTest {
9290
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
9391
uint64_t key;
9492
Slice in(iter->key());
93+
if (in == "" || in == "~") {
94+
// Ignore boundary keys.
95+
continue;
96+
}
9597
if (!ConsumeDecimalNumber(&in, &key) ||
9698
!in.empty() ||
9799
key < next_expected) {
@@ -233,7 +235,7 @@ TEST(CorruptionTest, TableFile) {
233235
dbi->TEST_CompactRange(1, NULL, NULL);
234236

235237
Corrupt(kTableFile, 100, 1);
236-
Check(99, 99);
238+
Check(90, 99);
237239
}
238240

239241
TEST(CorruptionTest, TableFileIndexData) {
@@ -299,40 +301,31 @@ TEST(CorruptionTest, CompactionInputError) {
299301
ASSERT_EQ(1, Property("leveldb.num-files-at-level" + NumberToString(last)));
300302

301303
Corrupt(kTableFile, 100, 1);
302-
Check(9, 9);
304+
Check(5, 9);
303305

304306
// Force compactions by writing lots of values
305307
Build(10000);
306308
Check(10000, 10000);
307309
}
308310

309311
TEST(CorruptionTest, CompactionInputErrorParanoid) {
310-
Options options;
311-
options.paranoid_checks = true;
312-
options.write_buffer_size = 1048576;
313-
Reopen(&options);
312+
options_.paranoid_checks = true;
313+
options_.write_buffer_size = 512 << 10;
314+
Reopen();
314315
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
315316

316-
// Fill levels >= 1 so memtable compaction outputs to level 1
317-
for (int level = 1; level < config::kNumLevels; level++) {
318-
dbi->Put(WriteOptions(), "", "begin");
319-
dbi->Put(WriteOptions(), "~", "end");
317+
// Make multiple inputs so we need to compact.
318+
for (int i = 0; i < 2; i++) {
319+
Build(10);
320320
dbi->TEST_CompactMemTable();
321+
Corrupt(kTableFile, 100, 1);
322+
env_.SleepForMicroseconds(100000);
321323
}
324+
dbi->CompactRange(NULL, NULL);
322325

323-
Build(10);
324-
dbi->TEST_CompactMemTable();
325-
ASSERT_EQ(1, Property("leveldb.num-files-at-level0"));
326-
327-
Corrupt(kTableFile, 100, 1);
328-
Check(9, 9);
329-
330-
// Write must eventually fail because of corrupted table
331-
Status s;
326+
// Write must fail because of corrupted table
332327
std::string tmp1, tmp2;
333-
for (int i = 0; i < 10000 && s.ok(); i++) {
334-
s = db_->Put(WriteOptions(), Key(i, &tmp1), Value(i, &tmp2));
335-
}
328+
Status s = db_->Put(WriteOptions(), Key(5, &tmp1), Value(5, &tmp2));
336329
ASSERT_TRUE(!s.ok()) << "write did not fail in corrupted paranoid db";
337330
}
338331

‎db/db_impl.cc

+27-14
Original file line numberDiff line numberDiff line change
@@ -113,14 +113,14 @@ Options SanitizeOptions(const std::string& dbname,
113113
return result;
114114
}
115115

116-
DBImpl::DBImpl(const Options& options, const std::string& dbname)
117-
: env_(options.env),
118-
internal_comparator_(options.comparator),
119-
internal_filter_policy_(options.filter_policy),
120-
options_(SanitizeOptions(
121-
dbname, &internal_comparator_, &internal_filter_policy_, options)),
122-
owns_info_log_(options_.info_log != options.info_log),
123-
owns_cache_(options_.block_cache != options.block_cache),
116+
DBImpl::DBImpl(const Options& raw_options, const std::string& dbname)
117+
: env_(raw_options.env),
118+
internal_comparator_(raw_options.comparator),
119+
internal_filter_policy_(raw_options.filter_policy),
120+
options_(SanitizeOptions(dbname, &internal_comparator_,
121+
&internal_filter_policy_, raw_options)),
122+
owns_info_log_(options_.info_log != raw_options.info_log),
123+
owns_cache_(options_.block_cache != raw_options.block_cache),
124124
dbname_(dbname),
125125
db_lock_(NULL),
126126
shutting_down_(NULL),
@@ -130,6 +130,7 @@ DBImpl::DBImpl(const Options& options, const std::string& dbname)
130130
logfile_(NULL),
131131
logfile_number_(0),
132132
log_(NULL),
133+
seed_(0),
133134
tmp_batch_(new WriteBatch),
134135
bg_compaction_scheduled_(false),
135136
manual_compaction_(NULL),
@@ -138,7 +139,7 @@ DBImpl::DBImpl(const Options& options, const std::string& dbname)
138139
has_imm_.Release_Store(NULL);
139140

140141
// Reserve ten files or so for other uses and give the rest to TableCache.
141-
const int table_cache_size = options.max_open_files - kNumNonTableCacheFiles;
142+
const int table_cache_size = options_.max_open_files - kNumNonTableCacheFiles;
142143
table_cache_ = new TableCache(dbname_, &options_, table_cache_size);
143144

144145
versions_ = new VersionSet(dbname_, &options_, table_cache_,
@@ -1027,7 +1028,8 @@ static void CleanupIteratorState(void* arg1, void* arg2) {
10271028
} // namespace
10281029

10291030
Iterator* DBImpl::NewInternalIterator(const ReadOptions& options,
1030-
SequenceNumber* latest_snapshot) {
1031+
SequenceNumber* latest_snapshot,
1032+
uint32_t* seed) {
10311033
IterState* cleanup = new IterState;
10321034
mutex_.Lock();
10331035
*latest_snapshot = versions_->LastSequence();
@@ -1051,13 +1053,15 @@ Iterator* DBImpl::NewInternalIterator(const ReadOptions& options,
10511053
cleanup->version = versions_->current();
10521054
internal_iter->RegisterCleanup(CleanupIteratorState, cleanup, NULL);
10531055

1056+
*seed = ++seed_;
10541057
mutex_.Unlock();
10551058
return internal_iter;
10561059
}
10571060

10581061
Iterator* DBImpl::TEST_NewInternalIterator() {
10591062
SequenceNumber ignored;
1060-
return NewInternalIterator(ReadOptions(), &ignored);
1063+
uint32_t ignored_seed;
1064+
return NewInternalIterator(ReadOptions(), &ignored, &ignored_seed);
10611065
}
10621066

10631067
int64_t DBImpl::TEST_MaxNextLevelOverlappingBytes() {
@@ -1114,12 +1118,21 @@ Status DBImpl::Get(const ReadOptions& options,
11141118

11151119
Iterator* DBImpl::NewIterator(const ReadOptions& options) {
11161120
SequenceNumber latest_snapshot;
1117-
Iterator* internal_iter = NewInternalIterator(options, &latest_snapshot);
1121+
uint32_t seed;
1122+
Iterator* iter = NewInternalIterator(options, &latest_snapshot, &seed);
11181123
return NewDBIterator(
1119-
&dbname_, env_, user_comparator(), internal_iter,
1124+
this, user_comparator(), iter,
11201125
(options.snapshot != NULL
11211126
? reinterpret_cast<const SnapshotImpl*>(options.snapshot)->number_
1122-
: latest_snapshot));
1127+
: latest_snapshot),
1128+
seed);
1129+
}
1130+
1131+
void DBImpl::RecordReadSample(Slice key) {
1132+
MutexLock l(&mutex_);
1133+
if (versions_->current()->RecordReadSample(key)) {
1134+
MaybeScheduleCompaction();
1135+
}
11231136
}
11241137

11251138
const Snapshot* DBImpl::GetSnapshot() {

‎db/db_impl.h

+8-1
Original file line numberDiff line numberDiff line change
@@ -59,13 +59,19 @@ class DBImpl : public DB {
5959
// file at a level >= 1.
6060
int64_t TEST_MaxNextLevelOverlappingBytes();
6161

62+
// Record a sample of bytes read at the specified internal key.
63+
// Samples are taken approximately once every config::kReadBytesPeriod
64+
// bytes.
65+
void RecordReadSample(Slice key);
66+
6267
private:
6368
friend class DB;
6469
struct CompactionState;
6570
struct Writer;
6671

6772
Iterator* NewInternalIterator(const ReadOptions&,
68-
SequenceNumber* latest_snapshot);
73+
SequenceNumber* latest_snapshot,
74+
uint32_t* seed);
6975

7076
Status NewDB();
7177

@@ -135,6 +141,7 @@ class DBImpl : public DB {
135141
WritableFile* logfile_;
136142
uint64_t logfile_number_;
137143
log::Writer* log_;
144+
uint32_t seed_; // For sampling.
138145

139146
// Queue of writers.
140147
std::deque<Writer*> writers_;

0 commit comments

Comments
 (0)