Skip to content

HBASE-27498: Added logic in ConnectionImplementation.getKeepAliveMasterService to avoid expensive rpc calls in synchronized block #4889

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
taklwu merged 7 commits into from
Dec 8, 2022
Merged

HBASE-27498: Added logic in ConnectionImplementation.getKeepAliveMasterService to avoid expensive rpc calls in synchronized block #4889

Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
eb23af3
HBASE-27498-Added a logic to cache master state so that all threads d…
vaijosh Nov 19, 2022
e3700d1
HBASE-27498-Fixed formatting errors using "mvn spotless:apply"
vaijosh Nov 19, 2022
f90f0f2
HBASE-27498 : Implemented Review comments 1.Default master state cach…
vaijosh Nov 25, 2022
9b2ef31
HBASE-27498 : Fixed Spotless formatting issue.
vaijosh Nov 25, 2022
be30d7c
HBASE-27498 : Fixed Spotless formatting issue.
vaijosh Nov 25, 2022
467395e
HBASE-27498 : Fixed flaky test TestConnectionImplementation
vaijosh Dec 2, 2022
fb56dba
HBASE-27498 : Minor changes to fix flaky test
vaijosh Dec 2, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 35 additions & 19 deletions hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.hbase.thirdparty.com.google.common.base.Supplier;
import org.apache.hbase.thirdparty.com.google.common.base.Suppliers;
import org.apache.hbase.thirdparty.com.google.common.base.Throwables;
import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.hbase.thirdparty.com.google.protobuf.BlockingRpcChannel;
Expand Down Expand Up @@ -169,6 +171,9 @@
@InterfaceAudience.Private
public class ConnectionImplementation implements ClusterConnection, Closeable {
public static final String RETRIES_BY_SERVER_KEY = "hbase.client.retries.by.server";

public static final String MASTER_STATE_CACHE_TIMEOUT_SEC =
"hbase.client.master.state.cache.timeout.sec";
private static final Logger LOG = LoggerFactory.getLogger(ConnectionImplementation.class);

// The mode tells if HedgedRead, LoadBalance mode is supported.
Expand Down Expand Up @@ -249,6 +254,9 @@ public class ConnectionImplementation implements ClusterConnection, Closeable {
/** lock guards against multiple threads trying to query the meta region at the same time */
private final ReentrantLock userRegionLock = new ReentrantLock();

/** A Guava based TTL Cache to cache state of the master i.e. master is running or not */
final Supplier<Boolean> cachedMasterStateSupplier;

private ChoreService choreService;

/**
Expand Down Expand Up @@ -382,6 +390,24 @@ public void newDead(ServerName sn) {
default:
// Doing nothing
}

this.cachedMasterStateSupplier = Suppliers.memoizeWithExpiration(() -> {
if (this.masterServiceState.getStub() == null) {
return false;
}
try {
LOG.info("Getting master state using rpc call");
return this.masterServiceState.isMasterRunning();
} catch (UndeclaredThrowableException e) {
// It's somehow messy, but we can receive exceptions such as
// java.net.ConnectException but they're not declared. So we catch it...
LOG.info("Master connection is not running anymore", e.getUndeclaredThrowable());
return false;
} catch (IOException se) {
LOG.warn("Checking master connection", se);
return false;
}
}, conf.getLong(MASTER_STATE_CACHE_TIMEOUT_SEC, 30), TimeUnit.SECONDS);
Comment thread
vaijosh marked this conversation as resolved.
Outdated
}

private void spawnRenewalChore(final UserGroupInformation user) {
Expand Down Expand Up @@ -1365,12 +1391,14 @@ private void resetMasterServiceState(final MasterServiceState mss) {
}

private MasterKeepAliveConnection getKeepAliveMasterService() throws IOException {
synchronized (masterLock) {
if (!isKeepAliveMasterConnectedAndRunning(this.masterServiceState)) {
MasterServiceStubMaker stubMaker = new MasterServiceStubMaker();
this.masterServiceState.stub = stubMaker.makeStub();
if (!isKeepAliveMasterConnectedAndRunning(this.masterServiceState)) {
synchronized (masterLock) {
if (!isKeepAliveMasterConnectedAndRunning(this.masterServiceState)) {
MasterServiceStubMaker stubMaker = new MasterServiceStubMaker();
this.masterServiceState.stub = stubMaker.makeStub();
}
resetMasterServiceState(this.masterServiceState);
}
resetMasterServiceState(this.masterServiceState);
}
// Ugly delegation just so we can add in a Close method.
final MasterProtos.MasterService.BlockingInterface stub = this.masterServiceState.stub;
Expand Down Expand Up @@ -1945,20 +1973,8 @@ private static void release(MasterServiceState mss) {
}

private boolean isKeepAliveMasterConnectedAndRunning(MasterServiceState mss) {
if (mss.getStub() == null) {
return false;
}
try {
return mss.isMasterRunning();
} catch (UndeclaredThrowableException e) {
// It's somehow messy, but we can receive exceptions such as
// java.net.ConnectException but they're not declared. So we catch it...
LOG.info("Master connection is not running anymore", e.getUndeclaredThrowable());
return false;
} catch (IOException se) {
LOG.warn("Checking master connection", se);
return false;
}
LOG.info("Getting master connection state from TTL Cache");
return cachedMasterStateSupplier.get();
}

void releaseMaster(MasterServiceState mss) {
Expand Down