diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java index 9b4c925ad3e..0726ad58d46 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java @@ -74,6 +74,8 @@ public class RocksDBColumnarKeyValueStorage private static final Logger LOG = LoggerFactory.getLogger(RocksDBColumnarKeyValueStorage.class); private static final String DEFAULT_COLUMN = "default"; private static final String NO_SPACE_LEFT_ON_DEVICE = "No space left on device"; + private static final int MAX_BUSY_RETRIES = 5; + private static final int BUSY_RETRY_MILLISECONDS_INTERVAL = 100; private static final int ROCKSDB_FORMAT_VERSION = 5; private static final long ROCKSDB_BLOCK_SIZE = 32768; private static final long ROCKSDB_BLOCKCACHE_SIZE_HIGH_SPEC = 1_073_741_824L; @@ -405,12 +407,37 @@ public synchronized void commit() throws StorageException { LOG.error(e.getMessage()); System.exit(0); } - throw new StorageException(e); + if (e.getStatus().getCode() == Status.Code.Busy) { + LOG.error(e.getMessage()); + commitRetry(MAX_BUSY_RETRIES); + } else { + throw new StorageException(e); + } } finally { close(); } } + private void commitRetry(final int retries) { + if (retries == 0) { + LOG.error("RocksDB Busy - Already retried {} times", MAX_BUSY_RETRIES); + } else { + try { + LOG.debug("Retries left: {}", retries - 1); + innerTx.rollback(); + metrics.getRollbackCount().inc(); + Thread.sleep(BUSY_RETRY_MILLISECONDS_INTERVAL); + innerTx.commit(); + } catch (final RocksDBException e) { + if (e.getStatus().getCode() == Status.Code.Busy) { + commitRetry(retries - 1); + } + } catch (final InterruptedException e) { + throw new RuntimeException(e); + } + } + } + @Override public void rollback() { try { diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/unsegmented/RocksDBTransaction.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/unsegmented/RocksDBTransaction.java index ffd5a1596f6..9a8ac286323 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/unsegmented/RocksDBTransaction.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/unsegmented/RocksDBTransaction.java @@ -20,6 +20,7 @@ import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBMetrics; import org.rocksdb.RocksDBException; +import org.rocksdb.Status; import org.rocksdb.Transaction; import org.rocksdb.WriteOptions; import org.slf4j.Logger; @@ -27,9 +28,10 @@ /** The RocksDb transaction. */ public class RocksDBTransaction implements KeyValueStorageTransaction { - private static final Logger logger = LoggerFactory.getLogger(RocksDBTransaction.class); + private static final Logger LOG = LoggerFactory.getLogger(RocksDBTransaction.class); private static final String NO_SPACE_LEFT_ON_DEVICE = "No space left on device"; - + private static final int MAX_BUSY_RETRIES = 5; + private static final int BUSY_RETRY_MILLISECONDS_INTERVAL = 100; private final RocksDBMetrics metrics; private final Transaction innerTx; private final WriteOptions options; @@ -54,7 +56,7 @@ public void put(final byte[] key, final byte[] value) { innerTx.put(key, value); } catch (final RocksDBException e) { if (e.getMessage().contains(NO_SPACE_LEFT_ON_DEVICE)) { - logger.error(e.getMessage()); + LOG.error(e.getMessage()); System.exit(0); } throw new StorageException(e); @@ -67,7 +69,7 @@ public void remove(final byte[] key) { innerTx.delete(key); } catch (final RocksDBException e) { if (e.getMessage().contains(NO_SPACE_LEFT_ON_DEVICE)) { - logger.error(e.getMessage()); + LOG.error(e.getMessage()); System.exit(0); } throw new StorageException(e); @@ -80,15 +82,39 @@ public void commit() throws StorageException { innerTx.commit(); } catch (final RocksDBException e) { if (e.getMessage().contains(NO_SPACE_LEFT_ON_DEVICE)) { - logger.error(e.getMessage()); + LOG.error(e.getMessage()); System.exit(0); } + if (e.getStatus().getCode() == Status.Code.Busy) { + LOG.error(e.getMessage()); + commitRetry(MAX_BUSY_RETRIES); + } throw new StorageException(e); } finally { close(); } } + private void commitRetry(final int retries) { + if (retries == 0) { + LOG.error("RocksDB Busy - Already retried {} times", MAX_BUSY_RETRIES); + } else { + try { + LOG.debug("Retries left: {}", retries - 1); + innerTx.rollback(); + metrics.getRollbackCount().inc(); + Thread.sleep(BUSY_RETRY_MILLISECONDS_INTERVAL); + innerTx.commit(); + } catch (final RocksDBException e) { + if (e.getStatus().getCode() == Status.Code.Busy) { + commitRetry(retries - 1); + } + } catch (final InterruptedException e) { + throw new RuntimeException(e); + } + } + } + @Override public void rollback() { try { @@ -96,7 +122,7 @@ public void rollback() { metrics.getRollbackCount().inc(); } catch (final RocksDBException e) { if (e.getMessage().contains(NO_SPACE_LEFT_ON_DEVICE)) { - logger.error(e.getMessage()); + LOG.error(e.getMessage()); System.exit(0); } throw new StorageException(e);