Skip to content

[SPARK-32350][CORE] Add batch-write on LevelDB to improve performance of HybridStore #29149

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
baohe-zhang wants to merge 2 commits into from
Closed

[SPARK-32350][CORE] Add batch-write on LevelDB to improve performance of HybridStore #29149

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
58023ff
Add writeAll() on LevelDB and use it on HybridStore
Jul 17, 2020
a12e178
Extract the duplicate code to a new method
Jul 19, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 61 additions & 12 deletions common/kvstore/src/main/java/org/apache/spark/util/kvstore/LevelDB.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import static java.nio.charset.StandardCharsets.UTF_8;

import com.google.common.annotations.VisibleForTesting;
Expand Down Expand Up @@ -153,24 +154,72 @@ public void write(Object value) throws Exception {
try (WriteBatch batch = db().createWriteBatch()) {
byte[] data = serializer.serialize(value);
synchronized (ti) {
Object existing;
try {
existing = get(ti.naturalIndex().entityKey(null, value), value.getClass());
} catch (NoSuchElementException e) {
existing = null;
}
updateBatch(batch, value, data, value.getClass(), ti.naturalIndex(), ti.indices());
db().write(batch);
}
}
}

public void writeAll(List<?> values) throws Exception {
Preconditions.checkArgument(values != null && !values.isEmpty(),
"Non-empty values required.");

// Group by class, in case there are values from different classes in the values
// Typical usecase is for this to be a single class.
// A NullPointerException will be thrown if values contain null object.
for (Map.Entry<? extends Class<?>, ? extends List<?>> entry :
values.stream().collect(Collectors.groupingBy(Object::getClass)).entrySet()) {

final Iterator<?> valueIter = entry.getValue().iterator();
final Iterator<byte[]> serializedValueIter;

// Deserialize outside synchronized block
List<byte[]> list = new ArrayList<>(entry.getValue().size());
for (Object value : values) {
list.add(serializer.serialize(value));
}
serializedValueIter = list.iterator();

final Class<?> klass = entry.getKey();
final LevelDBTypeInfo ti = getTypeInfo(klass);

PrefixCache cache = new PrefixCache(value);
byte[] naturalKey = ti.naturalIndex().toKey(ti.naturalIndex().getValue(value));
for (LevelDBTypeInfo.Index idx : ti.indices()) {
byte[] prefix = cache.getPrefix(idx);
idx.add(batch, value, existing, data, naturalKey, prefix);
synchronized (ti) {
final LevelDBTypeInfo.Index naturalIndex = ti.naturalIndex();
final Collection<LevelDBTypeInfo.Index> indices = ti.indices();

try (WriteBatch batch = db().createWriteBatch()) {
while (valueIter.hasNext()) {
updateBatch(batch, valueIter.next(), serializedValueIter.next(), klass,
naturalIndex, indices);
}
db().write(batch);
}
db().write(batch);
}
}
}

private void updateBatch(
WriteBatch batch,
Object value,
byte[] data,
Class<?> klass,
LevelDBTypeInfo.Index naturalIndex,
Collection<LevelDBTypeInfo.Index> indices) throws Exception {
Object existing;
try {
existing = get(naturalIndex.entityKey(null, value), klass);
} catch (NoSuchElementException e) {
existing = null;
}

PrefixCache cache = new PrefixCache(value);
byte[] naturalKey = naturalIndex.toKey(naturalIndex.getValue(value));
for (LevelDBTypeInfo.Index idx : indices) {
byte[] prefix = cache.getPrefix(idx);
idx.add(batch, value, existing, data, naturalKey, prefix);
}
}

@Override
public void delete(Class<?> type, Object naturalKey) throws Exception {
Preconditions.checkArgument(naturalKey != null, "Null keys are not allowed.");
Expand Down
9 changes: 5 additions & 4 deletions core/src/main/scala/org/apache/spark/deploy/history/HybridStore.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ import java.util.concurrent.atomic.AtomicBoolean

import scala.collection.JavaConverters._

import com.google.common.collect.Lists;

import org.apache.spark.util.kvstore._

/**
Expand Down Expand Up @@ -144,10 +146,9 @@ private[history] class HybridStore extends KVStore {
backgroundThread = new Thread(() => {
try {
for (klass <- klassMap.keys().asScala) {
val it = inMemoryStore.view(klass).closeableIterator()
while (it.hasNext()) {
levelDB.write(it.next())
}
val values = Lists.newArrayList(
Copy link
Contributor

@HeartSaVioR HeartSaVioR Jul 18, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This would be OK, given all entries are from inMemoryStore which are already materialized into memory.

inMemoryStore.view(klass).closeableIterator())
levelDB.writeAll(values)
}
listener.onSwitchToLevelDBSuccess()
shouldUseInMemoryStore.set(false)
Expand Down