RegionConfiguration encapsulates the configuration attributes for a Region to be
* created on the server.
*
- * @author Barry Oglesby
* @since 6.5
*/
@SuppressWarnings({"serial", "unchecked"})
diff --git a/extensions/geode-modules/src/main/java/com/gemstone/gemfire/modules/util/TouchPartitionedRegionEntriesFunction.java b/extensions/geode-modules/src/main/java/com/gemstone/gemfire/modules/util/TouchPartitionedRegionEntriesFunction.java
index d2b74c8094c4..15212b77ff32 100644
--- a/extensions/geode-modules/src/main/java/com/gemstone/gemfire/modules/util/TouchPartitionedRegionEntriesFunction.java
+++ b/extensions/geode-modules/src/main/java/com/gemstone/gemfire/modules/util/TouchPartitionedRegionEntriesFunction.java
@@ -31,7 +31,6 @@
/**
* Touches the keys contained in the set of keys by performing a get on the partitioned region.
*
- * @author Barry Oglesby
*/
public class TouchPartitionedRegionEntriesFunction implements Function, Declarable {
diff --git a/extensions/geode-modules/src/main/java/com/gemstone/gemfire/modules/util/TouchReplicatedRegionEntriesFunction.java b/extensions/geode-modules/src/main/java/com/gemstone/gemfire/modules/util/TouchReplicatedRegionEntriesFunction.java
index 735c27f4cacb..0e1561ea8752 100644
--- a/extensions/geode-modules/src/main/java/com/gemstone/gemfire/modules/util/TouchReplicatedRegionEntriesFunction.java
+++ b/extensions/geode-modules/src/main/java/com/gemstone/gemfire/modules/util/TouchReplicatedRegionEntriesFunction.java
@@ -30,7 +30,6 @@
* Touches the keys contained in the set of keys by performing a get on the replicated region. This is a non-data-aware
* function invoked using onMembers or onServers.
*
- * @author Barry Oglesby
*/
public class TouchReplicatedRegionEntriesFunction implements Function, Declarable {
diff --git a/extensions/geode-modules/src/test/java/com/gemstone/gemfire/modules/session/Tomcat6SessionsJUnitTest.java b/extensions/geode-modules/src/test/java/com/gemstone/gemfire/modules/session/Tomcat6SessionsJUnitTest.java
index 0ce73aec9d23..7500a3c58008 100644
--- a/extensions/geode-modules/src/test/java/com/gemstone/gemfire/modules/session/Tomcat6SessionsJUnitTest.java
+++ b/extensions/geode-modules/src/test/java/com/gemstone/gemfire/modules/session/Tomcat6SessionsJUnitTest.java
@@ -22,7 +22,6 @@
import org.junit.experimental.categories.Category;
/**
- * @author Jens Deppe
*/
@Category(UnitTest.class)
public class Tomcat6SessionsJUnitTest extends TestSessionsBase {
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/cache/client/internal/OpExecutorImpl.java b/gemfire-core/src/main/java/com/gemstone/gemfire/cache/client/internal/OpExecutorImpl.java
new file mode 100644
index 000000000000..af19584d9184
--- /dev/null
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/cache/client/internal/OpExecutorImpl.java
@@ -0,0 +1,972 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gemstone.gemfire.cache.client.internal;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.NotSerializableException;
+import java.net.ConnectException;
+import java.net.SocketException;
+import java.net.SocketTimeoutException;
+import java.nio.BufferUnderflowException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.logging.log4j.Logger;
+
+import com.gemstone.gemfire.CancelCriterion;
+import com.gemstone.gemfire.CancelException;
+import com.gemstone.gemfire.CopyException;
+import com.gemstone.gemfire.GemFireException;
+import com.gemstone.gemfire.SerializationException;
+import com.gemstone.gemfire.cache.CacheRuntimeException;
+import com.gemstone.gemfire.cache.RegionDestroyedException;
+import com.gemstone.gemfire.cache.SynchronizationCommitConflictException;
+import com.gemstone.gemfire.cache.TransactionException;
+import com.gemstone.gemfire.cache.client.NoAvailableServersException;
+import com.gemstone.gemfire.cache.client.ServerConnectivityException;
+import com.gemstone.gemfire.cache.client.ServerOperationException;
+import com.gemstone.gemfire.cache.client.ServerRefusedConnectionException;
+import com.gemstone.gemfire.cache.client.SubscriptionNotEnabledException;
+import com.gemstone.gemfire.cache.client.internal.ExecuteFunctionOp.ExecuteFunctionOpImpl;
+import com.gemstone.gemfire.cache.client.internal.ExecuteRegionFunctionOp.ExecuteRegionFunctionOpImpl;
+import com.gemstone.gemfire.cache.client.internal.QueueManager.QueueConnections;
+import com.gemstone.gemfire.cache.client.internal.pooling.ConnectionDestroyedException;
+import com.gemstone.gemfire.cache.client.internal.pooling.ConnectionManager;
+import com.gemstone.gemfire.cache.execute.FunctionException;
+import com.gemstone.gemfire.cache.execute.FunctionInvocationTargetException;
+import com.gemstone.gemfire.distributed.internal.ServerLocation;
+import com.gemstone.gemfire.internal.cache.PoolManagerImpl;
+import com.gemstone.gemfire.internal.cache.PutAllPartialResultException;
+import com.gemstone.gemfire.internal.cache.TXManagerImpl;
+import com.gemstone.gemfire.internal.cache.TXStateProxy;
+import com.gemstone.gemfire.internal.cache.execute.InternalFunctionInvocationTargetException;
+import com.gemstone.gemfire.internal.cache.tier.BatchException;
+import com.gemstone.gemfire.internal.cache.wan.BatchException70;
+import com.gemstone.gemfire.internal.logging.LogService;
+import com.gemstone.gemfire.internal.logging.log4j.LogMarker;
+import com.gemstone.gemfire.security.AuthenticationRequiredException;
+import com.gemstone.gemfire.security.GemFireSecurityException;
+
+/**
+ * Called from the client and execute client to server
+ * requests against servers. Handles retrying to different servers,
+ * and marking servers dead if we get exception from them.
+ * @since 5.7
+ */
+public class OpExecutorImpl implements ExecutablePool {
+ private static final Logger logger = LogService.getLogger();
+
+ private static final boolean TRY_SERVERS_ONCE = Boolean.getBoolean("gemfire.PoolImpl.TRY_SERVERS_ONCE");
+ private static final int TX_RETRY_ATTEMPT = Integer.getInteger("gemfire.txRetryAttempt", 500);
+
+ private final ConnectionManager connectionManager;
+ private final int retryAttempts;
+ private final long serverTimeout;
+ private final boolean threadLocalConnections;
+ private final ThreadLocalDistributionManager uses a {@link
+ * MembershipManager} to distribute {@link DistributionMessage messages}
+ * queued in {@link MQueue}s.
+ *
+ *
+ *
+ * Code that wishes to send a {@link DistributionMessage} must get
+ * the DistributionManager and invoke {@link
+ * #putOutgoing}.
+ *
+ *
+ *
+ * DistributionManager is not intended to be
+ * serialized. It is Externalizable only to prevent it
+ * from being copy shared. See {@link #writeExternal}.
+ *
+ *
+ *
+ * Prior to GemFire 4.0, DistributionManager was an
+ * abstract class with two concrete subclasses,
+ * LocalDistributionManager and
+ * ConsoleDistributionManager. We decided that
+ * ConsoleDistributionManager (which was used for the GUI
+ * console and admin APIs) did not offer enough interesting
+ * functionality to warrant a separate class. More importantly, it
+ * prevented the Cache and admin APIs from being used in the same VM.
+ * So, we refactored the code of those two subclasses into
+ * DistributionManager.
+ *
+ * @since 2.0
+ *
+ * @see DistributionMessage#process
+ * @see IgnoredByManager
+ */
+public class DistributionManager
+ implements Externalizable, DM {
+
+ private static final Logger logger = LogService.getLogger();
+
+ private static final boolean SYNC_EVENTS =
+ Boolean.getBoolean("DistributionManager.syncEvents");
+
+ /**
+ * WARNING: setting this to true may break dunit tests.
+ *
see com.gemstone.gemfire.cache30.ClearMultiVmCallBkDUnitTest
+ */
+ public static final boolean INLINE_PROCESS =
+ !Boolean.getBoolean("DistributionManager.enqueueOrderedMessages");
+
+ /** Flag indicating whether to use single Serial-Executor thread or
+ * Multiple Serial-executor thread,
+ */
+ public static final boolean MULTI_SERIAL_EXECUTORS =
+ !Boolean.getBoolean("DistributionManager.singleSerialExecutor");
+
+ /** The name of the distribution manager (identifies it in GemFire) */
+ public static final String NAME = "GemFire";
+
+ /** The number of milliseconds to wait for distribution-related
+ * things to happen */
+ public static final long TIMEOUT =
+ Long.getLong("DistributionManager.TIMEOUT", -1).longValue();
+
+ public static final int PUSHER_THREADS =
+ Integer.getInteger("DistributionManager.PUSHER_THREADS", 50).intValue();
+
+ public static final int PUSHER_QUEUE_SIZE =
+ Integer.getInteger("DistributionManager.PUSHER_QUEUE_SIZE", 4096).intValue();
+
+
+ public static final int MAX_WAITING_THREADS =
+ Integer.getInteger("DistributionManager.MAX_WAITING_THREADS", Integer.MAX_VALUE).intValue();
+
+ public static final int MAX_PR_META_DATA_CLEANUP_THREADS =
+ Integer.getInteger("DistributionManager.MAX_PR_META_DATA_CLEANUP_THREADS", 1).intValue();
+
+ public static final int MAX_THREADS = Integer.getInteger("DistributionManager.MAX_THREADS", 100).intValue();
+ public static final int MAX_PR_THREADS = Integer.getInteger("DistributionManager.MAX_PR_THREADS", Math.max(Runtime.getRuntime().availableProcessors()*4, 16)).intValue();
+ public static final int MAX_FE_THREADS = Integer.getInteger("DistributionManager.MAX_FE_THREADS", Math.max(Runtime.getRuntime().availableProcessors()*4, 16)).intValue();
+ // Integer.getInteger("DistributionManager.MAX_THREADS", max(Runtime.getRuntime().availableProcessors()*2, 2)).intValue();
+
+ public static final int INCOMING_QUEUE_LIMIT =
+ Integer.getInteger("DistributionManager.INCOMING_QUEUE_LIMIT", 80000).intValue();
+ public static final int INCOMING_QUEUE_THROTTLE =
+ Integer.getInteger("DistributionManager.INCOMING_QUEUE_THROTTLE", (int)(INCOMING_QUEUE_LIMIT * 0.75)).intValue();
+
+ /** Throttling based on the Queue byte size */
+ public static final double THROTTLE_PERCENT =
+ (double) (Integer.getInteger("DistributionManager.SERIAL_QUEUE_THROTTLE_PERCENT", 75).intValue())/100;
+ public static final int SERIAL_QUEUE_BYTE_LIMIT =
+ Integer.getInteger("DistributionManager.SERIAL_QUEUE_BYTE_LIMIT", (40 * (1024 * 1024))).intValue();
+ public static final int SERIAL_QUEUE_THROTTLE =
+ Integer.getInteger("DistributionManager.SERIAL_QUEUE_THROTTLE", (int)(SERIAL_QUEUE_BYTE_LIMIT * THROTTLE_PERCENT)).intValue();
+ public static final int TOTAL_SERIAL_QUEUE_BYTE_LIMIT =
+ Integer.getInteger("DistributionManager.TOTAL_SERIAL_QUEUE_BYTE_LIMIT", (80 * (1024 * 1024))).intValue();
+ public static final int TOTAL_SERIAL_QUEUE_THROTTLE =
+ Integer.getInteger("DistributionManager.TOTAL_SERIAL_QUEUE_THROTTLE", (int)(SERIAL_QUEUE_BYTE_LIMIT * THROTTLE_PERCENT)).intValue();
+
+ /** Throttling based on the Queue item size */
+ public static final int SERIAL_QUEUE_SIZE_LIMIT =
+ Integer.getInteger("DistributionManager.SERIAL_QUEUE_SIZE_LIMIT", 20000).intValue();
+ public static final int SERIAL_QUEUE_SIZE_THROTTLE =
+ Integer.getInteger("DistributionManager.SERIAL_QUEUE_SIZE_THROTTLE", (int)(SERIAL_QUEUE_SIZE_LIMIT * THROTTLE_PERCENT)).intValue();
+
+ /** Max number of serial Queue executors, in case of multi-serial-queue executor */
+ public static final int MAX_SERIAL_QUEUE_THREAD =
+ Integer.getInteger("DistributionManager.MAX_SERIAL_QUEUE_THREAD", 20).intValue();
+
+ /**
+ * Whether or not to include link local addresses in the list of addresses we use
+ * to determine if two members are no the same host.
+ *
+ * Added for normura issue 7033 - they have duplicate link local addresses on different boxes
+ */
+ public static volatile boolean INCLUDE_LINK_LOCAL_ADDRESSES =
+ Boolean.getBoolean("gemfire.IncludeLinkLocalAddresses");
+
+ /** The DM type for regular distribution managers */
+ public static final int NORMAL_DM_TYPE = 10;
+
+ /** The DM type for locator distribution managers
+ * @since 7.0
+ */
+ public static final int LOCATOR_DM_TYPE = 11;
+
+ /** The DM type for Console (admin-only) distribution managers */
+ public static final int ADMIN_ONLY_DM_TYPE = 12;
+
+ public static final int LONER_DM_TYPE = 13;
+
+ /**
+ * an NIO priority type
+ * @see com.gemstone.gemfire.distributed.internal.PooledDistributionMessage
+ * @see #SERIAL_EXECUTOR
+ * @see #HIGH_PRIORITY_EXECUTOR
+ * @see #WAITING_POOL_EXECUTOR
+ */
+ public static final int STANDARD_EXECUTOR = 73;
+
+ /**
+ * an NIO priority type
+ *
+ * @see com.gemstone.gemfire.distributed.internal.SerialDistributionMessage
+ * @see #STANDARD_EXECUTOR
+ */
+ public static final int SERIAL_EXECUTOR = 74;
+
+ /**
+ * an NIO priority type
+
+ * @see com.gemstone.gemfire.distributed.internal.HighPriorityDistributionMessage
+ * @see #STANDARD_EXECUTOR
+ */
+ public static final int HIGH_PRIORITY_EXECUTOR = 75;
+
+ // 76 not in use
+
+ /**
+ * an NIO priority type
+ *
+ * @see com.gemstone.gemfire.internal.cache.InitialImageOperation
+ * @see #STANDARD_EXECUTOR
+ */
+ public static final int WAITING_POOL_EXECUTOR = 77;
+
+ /**
+ * an NIO priority type
+ *
+ * @see com.gemstone.gemfire.internal.cache.InitialImageOperation
+ * @see #STANDARD_EXECUTOR
+ */
+ public static final int PARTITIONED_REGION_EXECUTOR = 78;
+
+
+ /**
+ * Executor for view related messages
+ *
+ * @see com.gemstone.gemfire.distributed.internal.membership.gms.messages.ViewAckMessage
+ * @see #STANDARD_EXECUTOR
+ */
+ public static final int VIEW_EXECUTOR = 79;
+
+
+ public static final int REGION_FUNCTION_EXECUTION_EXECUTOR = 80;
+
+ /** The number of open distribution managers in this VM */
+ private static int openDMs = 0;
+
+// /** The stack trace of the last time a console DM was opened */
+// private static Exception openStackTrace;
+
+ /** Is this VM dedicated to administration (like a GUI console or a
+ * JMX agent)? If so, then it creates {@link #ADMIN_ONLY_DM_TYPE}
+ * type distribution managers.
+ *
+ * @since 4.0
+ */
+ public static volatile boolean isDedicatedAdminVM = false;
+
+ /**
+ * Is this admin agent used for a command line console.
+ * This flag controls whether connect will throw
+ * an exception or just wait for a DS if one is not
+ * available. If true, we will throw an exception.
+ *
+ */
+ public static volatile boolean isCommandLineAdminVM = false;
+
+
+
+ ///////////////////// Instance Fields //////////////////////
+
+ /** The id of this distribution manager */
+ final protected InternalDistributedMember myid;
+
+ /** The distribution manager type of this dm; set in its constructor. */
+ private final int dmType;
+
+ /** The
+ * Must be synchronized using {@link #unfinishedStartupsLock}
+ */
+ private Set unfinishedStartups = null;
+
+ /**
+ * Synchronization for {@link #unfinishedStartups}
+ */
+ private final Object unfinishedStartupsLock = new Object();
+
+ public void setUnfinishedStartups(Collection s) {
+ synchronized (unfinishedStartupsLock) {
+ Assert.assertTrue(unfinishedStartups == null,
+ "Set unfinished startups twice");
+ unfinishedStartups = new HashSet(s);
+
+ // OK, I don't _quite_ trust the list to be current, so let's
+ // prune it here.
+ Iterator it = unfinishedStartups.iterator();
+ synchronized (this.membersLock) {
+ while (it.hasNext()) {
+ InternalDistributedMember m = (InternalDistributedMember)it.next();
+ if (!isCurrentMember(m)) {
+ it.remove();
+ }
+ } // while
+ } // synchronized
+ }
+ }
+
+ public void removeUnfinishedStartup(InternalDistributedMember m,
+ boolean departed) {
+ synchronized (unfinishedStartupsLock) {
+ if (logger.isDebugEnabled()) {
+ logger.debug("removeUnfinishedStartup for {} wtih {}", m, unfinishedStartups);
+ }
+ if (unfinishedStartups == null)
+ return; // not yet done with startup
+ if (!unfinishedStartups.remove(m))
+ return;
+ StringId msg = null;
+ if (departed) {
+ msg = LocalizedStrings.DistributionManager_STOPPED_WAITING_FOR_STARTUP_REPLY_FROM_0_BECAUSE_THE_PEER_DEPARTED_THE_VIEW;
+ }
+ else {
+ msg = LocalizedStrings.DistributionManager_STOPPED_WAITING_FOR_STARTUP_REPLY_FROM_0_BECAUSE_THE_REPLY_WAS_FINALLY_RECEIVED;
+ }
+ logger.info(LocalizedMessage.create(msg, m));
+ int numLeft = unfinishedStartups.size();
+ if (numLeft != 0) {
+ logger.info(LocalizedMessage.create(
+ LocalizedStrings.DistributionManager_STILL_AWAITING_0_RESPONSES_FROM_1,
+ new Object[] {Integer.valueOf(numLeft), unfinishedStartups}));
+ }
+ } // synchronized
+ }
+
+ /**
+ * Processes the first startup response.
+ *
+ * @see StartupResponseMessage#process
+ */
+ void processStartupResponse(InternalDistributedMember sender,
+ long otherCacheTime, String theRejectionMessage) {
+ removeUnfinishedStartup(sender, false);
+ synchronized (this) {
+ if (!this.receivedStartupResponse) {
+ this.receivedStartupResponse = true;
+ }
+ if (theRejectionMessage != null && this.rejectionMessage == null) {
+ // remember the first non-null rejection. This fixes bug 33266
+ this.rejectionMessage = theRejectionMessage;
+ }
+ }
+ }
+
+ /**
+ * Processes the first startup response.
+ *
+ * @see StartupResponseMessage#process
+ */
+ void processStartupResponse(InternalDistributedMember sender,
+ String theRejectionMessage) {
+ removeUnfinishedStartup(sender, false);
+ synchronized (this) {
+ if (!this.receivedStartupResponse) {
+ // only set the cacheTimeDelta once
+ this.receivedStartupResponse = true;
+ }
+ if (theRejectionMessage != null && this.rejectionMessage == null) {
+ // remember the first non-null rejection. This fixes bug 33266
+ this.rejectionMessage = theRejectionMessage;
+ }
+ }
+ }
+
+ /**
+ * Based on a recent JGroups view, return a member that might be the
+ * next elder.
+ * @return the elder candidate, possibly this VM.
+ */
+ private InternalDistributedMember getElderCandidate() {
+ List
+ * This tryLock is attempted and if it fails, an exception is thrown to
+ * cause a Doug Lea style back-off (p. 149). It throws an exception because
+ * it needs to back down a couple of packages and I didn't want to couple
+ * this pkg too tightly with the dlock pkg.
+ *
+ * GrantorRequestProcessor catches the exception, releases and reacquires
+ * the Collaboration, and then comes back here to attempt the tryLock
+ * again. Currently nothing will stop it from re-attempting forever. It
+ * has to get the ElderState and cannot give up, but it can free up the
+ * Collaboration and then re-enter it. The other thread holding the
+ * elder lock will hold it only briefly. I've added a volatile called
+ * elderStateInitialized which should cause this back-off to occur only
+ * once in the life of a vm... once the elder, always the elder.
+ *
+ * TODO: Collaboration lock is no longer used. Do we need to to use tryLock?
+ */
+ private ElderState getElderStateWithTryLock(boolean useTryLock) {
+ boolean locked = false;
+ if (useTryLock) {
+ boolean interrupted = Thread.interrupted();
+ try {
+ locked = this.elderLock.tryLock(2000);
+ }
+ catch (InterruptedException e) {
+ interrupted = true;
+ getCancelCriterion().checkCancelInProgress(e);
+ // one last attempt and then allow it to fail for back-off...
+ locked = this.elderLock.tryLock();
+ }
+ finally {
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ } else {
+ locked = true;
+ this.elderLock.lock();
+ }
+ if (!locked) {
+ // try-lock must have failed
+ throw new IllegalStateException(LocalizedStrings.DistributionManager_POSSIBLE_DEADLOCK_DETECTED.toLocalizedString());
+ }
+ try {
+ if (this.elderState == null) {
+ this.elderState = new ElderState(this);
+ }
+ }
+ finally {
+ this.elderLock.unlock();
+ }
+ this.elderStateInitialized = true;
+// if (Thread.currentThread().isInterrupted())
+// throw new RuntimeException("Interrupted");
+ return this.elderState;
+ }
+
+ /**
+ * Waits until elder if newElder or newElder is no longer a member
+ * @return true if newElder is the elder; false if he is no longer a member
+ * or we are the elder.
+ */
+ public boolean waitForElder(final InternalDistributedMember desiredElder) {
+ MembershipListener l = null;
+ try {
+// Assert.assertTrue(
+// desiredElder.getVmKind() != DistributionManager.ADMIN_ONLY_DM_TYPE);
+ synchronized (this.elderMonitor) {
+ while (true) {
+ if (closeInProgress)
+ return false;
+ InternalDistributedMember currentElder = this.elder;
+// Assert.assertTrue(
+// currentElder.getVmKind() != DistributionManager.ADMIN_ONLY_DM_TYPE);
+ if (desiredElder.equals(currentElder)) {
+ return true;
+ }
+ if (!isCurrentMember(desiredElder)) {
+ return false; // no longer present
+ }
+ if (this.myid.equals(currentElder)) {
+ // Once we become the elder we no longer allow anyone else to be the
+ // elder so don't let them wait anymore.
+ return false;
+ }
+ if (l == null) {
+ l = new MembershipListener() {
+ public void memberJoined(InternalDistributedMember theId) {
+ // nothing needed
+ }
+ public void memberDeparted(InternalDistributedMember theId, boolean crashed) {
+ if (desiredElder.equals(theId)) {
+ notifyElderChangeWaiters();
+ }
+ }
+ public void memberSuspect(InternalDistributedMember id,
+ InternalDistributedMember whoSuspected, String reason) {
+ }
+ public void viewInstalled(NetView view) {
+ }
+ public void quorumLost(Set
+ This method was added to allow hydra to obtain thread-local
+ data for transport from one thread to another. */
+ public MembershipManager getMembershipManager() {
+ // NOTE: do not add cancellation checks here. This method is
+ // used during auto-reconnect after the DS has been closed
+ return membershipManager;
+ }
+
+
+ ////////////////////// Inner Classes //////////////////////
+
+
+ /**
+ * This class is used for DM's multi serial executor.
+ * The serial messages are managed/executed by multiple serial thread.
+ * This class takes care of executing messages related to a sender
+ * using the same thread.
+ */
+ static private class SerialQueuedExecutorPool {
+ /** To store the serial threads */
+ ConcurrentMap serialQueuedExecutorMap = new ConcurrentHashMap(MAX_SERIAL_QUEUE_THREAD);
+
+ /** To store the queue associated with thread */
+ Map serialQueuedMap = new HashMap(MAX_SERIAL_QUEUE_THREAD);
+
+ /** Holds mapping between sender to the serial thread-id */
+ Map senderToSerialQueueIdMap = new HashMap();
+
+ /** Holds info about unused thread, a thread is marked unused when the
+ * member associated with it has left distribution system.
+ */
+ ArrayList threadMarkedForUse = new ArrayList();
+
+ DistributionStats stats;
+ ThreadGroup threadGroup;
+
+ final boolean throttlingDisabled;
+
+ /**
+ * Constructor.
+ * @param group thread group to which the threads will belog to.
+ * @param stats
+ */
+ SerialQueuedExecutorPool(ThreadGroup group, DistributionStats stats, boolean throttlingDisabled) {
+ this.threadGroup = group;
+ this.stats = stats;
+ this.throttlingDisabled = throttlingDisabled;
+ }
+
+ /*
+ * Returns an id of the thread in serialQueuedExecutorMap, thats mapped to the
+ * given seder.
+ *
+ * @param sender
+ * @param createNew boolean flag to indicate whether to create a new id, if id
+ * doesnot exists.
+ */
+ private Integer getQueueId(InternalDistributedMember sender, boolean createNew) {
+ // Create a new Id.
+ Integer queueId;
+
+ synchronized (senderToSerialQueueIdMap)
+ {
+ // Check if there is a executor associated with this sender.
+ queueId = (Integer)senderToSerialQueueIdMap.get(sender);
+
+ if (!createNew || queueId != null){
+ return queueId;
+ }
+
+ // Create new.
+ // Check if any threads are availabe that is marked for Use.
+ if (!threadMarkedForUse.isEmpty()){
+ queueId = (Integer)threadMarkedForUse.remove(0);
+ }
+ // If Map is full, use the threads in round-robin fashion.
+ if (queueId == null){
+ queueId = Integer.valueOf((serialQueuedExecutorMap.size() + 1) % MAX_SERIAL_QUEUE_THREAD);
+ }
+ senderToSerialQueueIdMap.put(sender, queueId);
+ }
+ return queueId;
+ }
+
+ /*
+ * Returns the queue associated with this sender.
+ * Used in FlowControl for throttling (based on queue size).
+ */
+ public OverflowQueueWithDMStats getSerialQueue(InternalDistributedMember sender) {
+ Integer queueId = getQueueId(sender, false);
+ if (queueId == null){
+ return null;
+ }
+ return (OverflowQueueWithDMStats)serialQueuedMap.get(queueId);
+ }
+
+ /*
+ * Returns the serial queue executor, before returning the thread this
+ * applies throttling, based on the total serial queue size (total - sum
+ * of all the serial queue size).
+ * The throttling is applied during put event, this doesnt block the extract
+ * operation on the queue.
+ *
+ */
+ public SerialQueuedExecutorWithDMStats getThrottledSerialExecutor(InternalDistributedMember sender) {
+ SerialQueuedExecutorWithDMStats executor = getSerialExecutor(sender);
+
+ // Get the total serial queue size.
+ int totalSerialQueueMemSize = stats.getSerialQueueBytes();
+
+ // for tcp socket reader threads, this code throttles the thread
+ // to keep the sender-side from overwhelming the receiver.
+ // UDP readers are throttled in the FC protocol, which queries
+ // the queue to see if it should throttle
+ if (stats.getSerialQueueBytes() > TOTAL_SERIAL_QUEUE_THROTTLE &&
+ !DistributionMessage.isPreciousThread())
+ {
+ do {
+ boolean interrupted = Thread.interrupted();
+ try {
+ float throttlePercent = (float)(totalSerialQueueMemSize - TOTAL_SERIAL_QUEUE_THROTTLE) / (float)(TOTAL_SERIAL_QUEUE_BYTE_LIMIT - TOTAL_SERIAL_QUEUE_THROTTLE);
+ int sleep = (int)(100.0 * throttlePercent);
+ sleep = Math.max(sleep, 1);
+ Thread.sleep(sleep);
+ } catch (InterruptedException ex) {
+ interrupted = true;
+ // FIXME-InterruptedException
+ // Perhaps we should return null here?
+ }
+ finally {
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ this.stats.getSerialQueueHelper().incThrottleCount();
+ } while (stats.getSerialQueueBytes() >= TOTAL_SERIAL_QUEUE_BYTE_LIMIT);
+ }
+ return executor;
+ }
+
+ /*
+ * Returns the serial queue executor for the given sender.
+ */
+ public SerialQueuedExecutorWithDMStats getSerialExecutor(InternalDistributedMember sender) {
+ SerialQueuedExecutorWithDMStats executor = null;
+ Integer queueId = getQueueId(sender, true);
+ if ((executor = (SerialQueuedExecutorWithDMStats)serialQueuedExecutorMap.get(queueId)) != null){
+ return executor;
+ }
+ // If executor doesn't exists for this sender, create one.
+ executor = createSerialExecutor(queueId);
+
+ serialQueuedExecutorMap.put(queueId, executor);
+
+ if (logger.isDebugEnabled()){
+ logger.debug("Created Serial Queued Executor With queueId {}. Total number of live Serial Threads :{}",
+ queueId, serialQueuedExecutorMap.size());
+ }
+ stats.incSerialPooledThread();
+ return executor;
+ }
+
+ /*
+ * Creates a serial queue executor.
+ */
+ private SerialQueuedExecutorWithDMStats createSerialExecutor(final Integer id) {
+
+ BlockingQueue poolQueue;
+
+ if (SERIAL_QUEUE_BYTE_LIMIT == 0 || this.throttlingDisabled) {
+ poolQueue = new OverflowQueueWithDMStats(stats.getSerialQueueHelper());
+ } else {
+ poolQueue = new ThrottlingMemLinkedQueueWithDMStats(SERIAL_QUEUE_BYTE_LIMIT, SERIAL_QUEUE_THROTTLE, SERIAL_QUEUE_SIZE_LIMIT, SERIAL_QUEUE_SIZE_THROTTLE, this.stats.getSerialQueueHelper());
+ }
+
+ serialQueuedMap.put(id, poolQueue);
+
+ ThreadFactory tf = new ThreadFactory() {
+ public Thread newThread(final Runnable command) {
+ SerialQueuedExecutorPool.this.stats.incSerialPooledThreadStarts();
+ final Runnable r = new Runnable() {
+ public void run() {
+ ConnectionTable.threadWantsSharedResources();
+ Connection.makeReaderThread();
+ try {
+ command.run();
+ } finally {
+ ConnectionTable.releaseThreadsSockets();
+ }
+ }
+ };
+
+ Thread thread = new Thread(threadGroup, r, "Pooled Serial Message Processor " + id);
+ thread.setDaemon(true);
+ return thread;
+ }
+ };
+ return new SerialQueuedExecutorWithDMStats(poolQueue, this.stats.getSerialPooledProcessorHelper(), tf);
+ }
+
+ /*
+ * Does cleanup relating to this member. And marks the serial executor associated
+ * with this member for re-use.
+ */
+ public void handleMemberDeparture(InternalDistributedMember member)
+ {
+ Integer queueId = getQueueId(member, false);
+ if (queueId == null){
+ return;
+ }
+
+ boolean isUsed = false;
+
+ synchronized (senderToSerialQueueIdMap)
+ {
+ senderToSerialQueueIdMap.remove(member);
+
+ // Check if any other members are using the same executor.
+ for (Iterator iter = senderToSerialQueueIdMap.values().iterator(); iter.hasNext();) {
+ Integer value = (Integer)iter.next();
+ if (value.equals(queueId))
+ {
+ isUsed = true;
+ break;
+ }
+ }
+
+ // If not used mark this as unused.
+ if (!isUsed)
+ {
+ if (logger.isInfoEnabled(LogMarker.DM))
+ logger.info(LogMarker.DM, LocalizedMessage.create(
+ LocalizedStrings.DistributionManager_MARKING_THE_SERIALQUEUEDEXECUTOR_WITH_ID__0__USED_BY_THE_MEMBER__1__TO_BE_UNUSED,
+ new Object[] {queueId, member}));
+
+ threadMarkedForUse.add(queueId);
+ }
+ }
+ }
+
+ public void awaitTermination(long time, TimeUnit unit) throws InterruptedException {
+ long timeNanos = unit.toNanos(time);
+ long remainingNanos = timeNanos;
+ long start = System.nanoTime();
+ for (Iterator iter = serialQueuedExecutorMap.values().iterator(); iter.hasNext();) {
+ ExecutorService executor = (ExecutorService)iter.next();
+ executor.awaitTermination(remainingNanos, TimeUnit.NANOSECONDS);
+ remainingNanos = timeNanos = (System.nanoTime() - start);
+ if(remainingNanos <= 0) {
+ return;
+ }
+ }
+ }
+
+ protected void shutdown(){
+ for (Iterator iter = serialQueuedExecutorMap.values().iterator(); iter.hasNext();) {
+ ExecutorService executor = (ExecutorService)iter.next();
+ executor.shutdown();
+ }
+ }
+ }
+
+ /**
+ * A simple class used for locking the list of members of the
+ * distributed system. We give this lock its own class so that it
+ * shows up nicely in stack traces.
+ */
+ private static final class MembersLock {
+ protected MembersLock() {
+
+ }
+ }
+
+ /**
+ * A simple class used for locking the list of membership listeners.
+ * We give this lock its own class so that it shows up nicely in
+ * stack traces.
+ */
+ private static final class MembershipListenersLock {
+ protected MembershipListenersLock() {
+ }
+ }
+
+ /**
+ * This is the listener implementation for responding from events from
+ * the Membership Manager.
+ *
+ */
+ private final class MyListener implements DistributedMembershipListener {
+ DistributionManager dm;
+
+ public MyListener(DistributionManager dm) {
+ this.dm = dm;
+ }
+
+ public boolean isShutdownMsgSent() {
+ return shutdownMsgSent;
+ }
+
+ public void membershipFailure(String reason, Throwable t) {
+ exceptionInThreads = true;
+ DistributionManager.this.rootCause = t;
+ getSystem().disconnect(reason, t, true);
+ }
+
+ public void messageReceived(DistributionMessage message) {
+ handleIncomingDMsg(message);
+ }
+
+ public void newMemberConnected(InternalDistributedMember member) {
+ // Do not elect the elder here as surprise members invoke this callback
+ // without holding the view lock. That can cause a race condition and
+ // subsequent deadlock (#45566). Elder selection is now done when a view
+ // is installed.
+ dm.addNewMember(member);
+ }
+
+ public void memberDeparted(InternalDistributedMember theId, boolean crashed, String reason) {
+ boolean wasAdmin = getAdminMemberSet().contains(theId);
+ if (wasAdmin) {
+ // Pretend we received an AdminConsoleDisconnectMessage from the console that
+ // is no longer in the JavaGroup view.
+ // He must have died without sending a ShutdownMessage.
+ // This fixes bug 28454.
+ AdminConsoleDisconnectMessage message = new AdminConsoleDisconnectMessage();
+ message.setSender(theId);
+ message.setCrashed(crashed);
+ message.setAlertListenerExpected(true);
+ message.setIgnoreAlertListenerRemovalFailure(true); // we don't know if it was a listener so don't issue a warning
+ message.setRecipient(myid);
+ message.setReason(reason); //added for #37950
+ handleIncomingDMsg(message);
+ }
+ dm.handleManagerDeparture(theId, crashed, reason);
+ }
+
+ public void memberSuspect(InternalDistributedMember suspect, InternalDistributedMember whoSuspected, String reason) {
+ dm.handleManagerSuspect(suspect, whoSuspected, reason);
+ }
+
+ public void viewInstalled(NetView view) {
+ processElderSelection();
+ dm.handleViewInstalled(view);
+ }
+
+ /** this is invoked when quorum is being lost, before the view has been installed */
+ public void quorumLost(Set
+ * Keys are instances of {@link InternalDistributedMember}, values are
+ * Longs indicating the time this member was shunned.
+ *
+ * Members are removed when a view containing them is processed. If,
+ * after {@link #surpriseMemberTimeout} milliseconds have passed, a view
+ * containing the member has not arrived, the member is removed from
+ * membership and member-left notification is performed.
+ * >
+ * Accesses to this list needs to be under the read or write lock of {@link #latestViewLock}
+ *
+ * @see System#currentTimeMillis()
+ */
+ protected final Map
+ * Must be called with {@link #latestViewLock} held. Waits
+ * until there is a stable view. If the member has already
+ * been added, simply returns; else adds the member.
+ *
+ * @param dm the member joining
+ */
+ public boolean addSurpriseMember(DistributedMember dm) {
+ final InternalDistributedMember member = (InternalDistributedMember)dm;
+ boolean warn = false;
+
+ latestViewLock.writeLock().lock();
+ try {
+ // At this point, the join may have been discovered by
+ // other means.
+ if (latestView.contains(member)) {
+ return true;
+ }
+ if (surpriseMembers.containsKey(member)) {
+ return true;
+ }
+ if (member.getVmViewId() < 0) {
+ logger.warn("adding a surprise member that has not yet joined the distributed system: " + member, new Exception("stack trace"));
+ }
+ if (latestView.getViewId() > member.getVmViewId()) {
+ // tell the process that it should shut down distribution.
+ // Run in a separate thread so we don't hold the view lock during the request. Bug #44995
+ new Thread(Thread.currentThread().getThreadGroup(),
+ "Removing shunned GemFire node " + member) {
+ @Override
+ public void run() {
+ // fix for bug #42548
+ // this is an old member that shouldn't be added
+ logger.warn(LocalizedMessage.create(
+ LocalizedStrings.GroupMembershipService_Invalid_Surprise_Member, new Object[]{member, latestView}));
+ requestMemberRemoval(member, "this member is no longer in the view but is initiating connections");
+ }
+ }.start();
+ addShunnedMember(member);
+ return false;
+ }
+
+ // Adding him to this set ensures we won't remove him if a new
+ // view comes in and he's still not visible.
+ surpriseMembers.put(member, Long.valueOf(System.currentTimeMillis()));
+
+ if (shutdownInProgress()) {
+ // Force disconnect, esp. the TCPConduit
+ String msg = LocalizedStrings.GroupMembershipService_THIS_DISTRIBUTED_SYSTEM_IS_SHUTTING_DOWN.toLocalizedString();
+ if (directChannel != null) {
+ try {
+ directChannel.closeEndpoint(member, msg);
+ } catch (DistributedSystemDisconnectedException e) {
+ // ignore - happens during shutdown
+ }
+ }
+ destroyMember(member, false, msg); // for good luck
+ return true; // allow during shutdown
+ }
+
+ if (isShunned(member)) {
+ warn = true;
+ surpriseMembers.remove(member);
+ } else {
+
+ // Now that we're sure the member is new, add them.
+ // make sure the surprise-member cleanup task is running
+ if (this.cleanupTimer == null) {
+ startCleanupTimer();
+ } // cleanupTimer == null
+
+ // Ensure that the member is accounted for in the view
+ // Conjure up a new view including the new member. This is necessary
+ // because we are about to tell the listener about a new member, so
+ // the listener should rightfully expect that the member is in our
+ // membership view.
+
+ // However, we put the new member at the end of the list. This
+ // should ensure he's not chosen as an elder.
+ // This will get corrected when he finally shows up in the
+ // view.
+ NetView newMembers = new NetView(latestView, latestView.getViewId());
+ newMembers.add(member);
+ latestView = newMembers;
+ }
+ } finally {
+ latestViewLock.writeLock().unlock();
+ }
+ if (warn) { // fix for bug #41538 - deadlock while alerting
+ logger.warn(LocalizedMessage.create(
+ LocalizedStrings.GroupMembershipService_MEMBERSHIP_IGNORING_SURPRISE_CONNECT_FROM_SHUNNED_MEMBER_0, member));
+ } else {
+ listener.newMemberConnected(member);
+ }
+ return !warn;
+ }
+
+
+ /** starts periodic task to perform cleanup chores such as expire surprise members */
+ private void startCleanupTimer() {
+ latestViewLock.writeLock().lock();
+ try {
+ if (this.cleanupTimer != null) {
+ return;
+ }
+ DistributedSystem ds = InternalDistributedSystem.getAnyInstance();
+ if (ds != null && ds.isConnected()) {
+ this.cleanupTimer = new SystemTimer(ds, true);
+ SystemTimer.SystemTimerTask st = new SystemTimer.SystemTimerTask() {
+ @Override
+ public void run2() {
+ latestViewLock.writeLock().lock();
+ try {
+ long oldestAllowed = System.currentTimeMillis() - surpriseMemberTimeout;
+ for (Iterator it=surpriseMembers.entrySet().iterator(); it.hasNext(); ) {
+ Map.Entry entry = (Map.Entry)it.next();
+ Long birthtime = (Long)entry.getValue();
+ if (birthtime.longValue() < oldestAllowed) {
+ it.remove();
+ InternalDistributedMember m = (InternalDistributedMember)entry.getKey();
+ logger.info(LocalizedMessage.create(
+ LocalizedStrings.GroupMembershipService_MEMBERSHIP_EXPIRING_MEMBERSHIP_OF_SURPRISE_MEMBER_0, m));
+ removeWithViewLock(m, true, "not seen in membership view in "
+ + surpriseMemberTimeout + "ms");
+ }
+ }
+ } finally {
+ latestViewLock.writeLock().unlock();
+ }
+ }
+ };
+ this.cleanupTimer.scheduleAtFixedRate(st, surpriseMemberTimeout, surpriseMemberTimeout/3);
+ } // ds != null && ds.isConnected()
+ } finally {
+ latestViewLock.writeLock().unlock();
+ }
+ }
+ /**
+ * Dispatch the distribution message, or place it on the startup queue.
+ *
+ * @param msg the message to process
+ */
+ protected void handleOrDeferMessage(DistributionMessage msg) {
+ synchronized(startupLock) {
+ if (!processingEvents) {
+ startupMessages.add(new StartupEvent(msg));
+ return;
+ }
+ }
+ dispatchMessage(msg);
+ }
+
+ public void warnShun(DistributedMember m) {
+ latestViewLock.writeLock().lock();
+ try {
+ if (!shunnedMembers.containsKey(m)) {
+ return; // not shunned
+ }
+ if (shunnedAndWarnedMembers.contains(m)) {
+ return; // already warned
+ }
+ shunnedAndWarnedMembers.add(m);
+ } finally {
+ latestViewLock.writeLock().unlock();
+ }
+ // issue warning outside of sync since it may cause messaging and we don't
+ // want to hold the view lock while doing that
+ logger.warn(LocalizedMessage.create(LocalizedStrings.GroupMembershipService_MEMBERSHIP_DISREGARDING_SHUNNED_MEMBER_0, m));
+ }
+
+ @Override
+ public void processMessage(DistributionMessage msg) {
+ handleOrDeferMessage(msg);
+ }
+
+ /**
+ * Logic for processing a distribution message.
+ *
+ * It is possible to receive messages not consistent with our view.
+ * We handle this here, and generate an uplevel event if necessary
+ * @param msg the message
+ */
+ public void dispatchMessage(DistributionMessage msg) {
+ boolean isNew = false;
+ InternalDistributedMember m = msg.getSender();
+ boolean shunned = false;
+
+ // First grab the lock: check the sender against our stabilized view.
+ latestViewLock.writeLock().lock();
+ try {
+ if (isShunned(m)) {
+ if (msg instanceof StartupMessage) {
+ endShun(m);
+ }
+ else {
+ // fix for bug 41538 - sick alert listener causes deadlock
+ // due to view lock being held during messaging
+ shunned = true;
+ }
+ } // isShunned
+
+ if (!shunned) {
+ isNew = !latestView.contains(m) && !surpriseMembers.containsKey(m);
+
+ // If it's a new sender, wait our turn, generate the event
+ if (isNew) {
+ shunned = !addSurpriseMember(m);
+ } // isNew
+ }
+
+ // Latch the view before we unlock
+ } finally {
+ latestViewLock.writeLock().unlock();
+ }
+
+ if (shunned) { // bug #41538 - shun notification must be outside synchronization to avoid hanging
+ warnShun(m);
+ logger.info("Membership: Ignoring message from shunned member <{}>:{}", m, msg);
+ throw new MemberShunnedException(m);
+ }
+
+ listener.messageReceived(msg);
+ }
+
+ /**
+ * Process a new view object, or place on the startup queue
+ * @param viewArg the new view
+ */
+ protected void handleOrDeferViewEvent(NetView viewArg) {
+ if (this.isJoining) {
+ // bug #44373 - queue all view messages while joining.
+ // This is done under the latestViewLock, but we can't block here because
+ // we're sitting in the UDP reader thread.
+ synchronized(startupLock) {
+ startupMessages.add(new StartupEvent(viewArg));
+ return;
+ }
+ }
+ latestViewLock.writeLock().lock();
+ try {
+ synchronized(startupLock) {
+ if (!processingEvents) {
+ startupMessages.add(new StartupEvent(viewArg));
+ return;
+ }
+ }
+ // view processing can take a while, so we use a separate thread
+ // to avoid blocking a reader thread
+ NetView newView = viewArg;
+ long newId = viewArg.getViewId();
+ LocalViewMessage v = new LocalViewMessage(address, newId, newView,
+ GMSMembershipManager.this);
+
+ listener.messageReceived(v);
+ } finally {
+ latestViewLock.writeLock().unlock();
+ }
+ }
+
+ @Override
+ public void memberSuspected(InternalDistributedMember initiator, InternalDistributedMember suspect, String reason) {
+ SuspectMember s = new SuspectMember(initiator, suspect, reason);
+ handleOrDeferSuspect(s);
+ }
+
+ /**
+ * Process a new view object, or place on the startup queue
+ * @param suspectInfo the suspectee and suspector
+ */
+ protected void handleOrDeferSuspect(SuspectMember suspectInfo) {
+ latestViewLock.writeLock().lock();
+ try {
+ synchronized(startupLock) {
+ if (!processingEvents) {
+ return;
+ }
+ }
+ InternalDistributedMember suspect = suspectInfo.suspectedMember;
+ InternalDistributedMember who = suspectInfo.whoSuspected;
+ this.suspectedMembers.put(suspect, Long.valueOf(System.currentTimeMillis()));
+ try {
+ listener.memberSuspect(suspect, who, suspectInfo.reason);
+ }
+ catch (DistributedSystemDisconnectedException se) {
+ // let's not get huffy about it
+ }
+ } finally {
+ latestViewLock.writeLock().unlock();
+ }
+ }
+
+ /**
+ * Process a potential direct connect. Does not use
+ * the startup queue. It grabs the {@link #latestViewLock}
+ * and then processes the event.
+ *
+ * It is a potential event, because we don't know until we've
+ * grabbed a stable view if this is really a new member.
+ *
+ * @param member
+ */
+ private void processSurpriseConnect(
+ InternalDistributedMember member)
+ {
+ addSurpriseMember(member);
+ }
+
+ /**
+ * Dispatch routine for processing a single startup event
+ * @param o the startup event to handle
+ */
+ private void processStartupEvent(StartupEvent o) {
+ // Most common events first
+
+ if (o.isDistributionMessage()) { // normal message
+ try {
+ dispatchMessage(o.dmsg);
+ }
+ catch (MemberShunnedException e) {
+ // message from non-member - ignore
+ }
+ }
+ else if (o.isGmsView()) { // view event
+ processView(o.gmsView.getViewId(), o.gmsView);
+ }
+ else if (o.isSurpriseConnect()) { // connect
+ processSurpriseConnect(o.member);
+ }
+
+ else // sanity
+ throw new InternalGemFireError(LocalizedStrings.GroupMembershipService_UNKNOWN_STARTUP_EVENT_0.toLocalizedString(o));
+ }
+
+ /**
+ * Special mutex to create a critical section for
+ * {@link #startEventProcessing()}
+ */
+ private final Object startupMutex = new Object();
+
+
+ public void startEventProcessing()
+ {
+ // Only allow one thread to perform the work
+ synchronized (startupMutex) {
+ if (logger.isDebugEnabled())
+ logger.debug("Membership: draining startup events.");
+ // Remove the backqueue of messages, but allow
+ // additional messages to be added.
+ for (;;) {
+ StartupEvent ev;
+ // Only grab the mutex while reading the queue.
+ // Other events may arrive while we're attempting to
+ // drain the queue. This is OK, we'll just keep processing
+ // events here until we've caught up.
+ synchronized (startupLock) {
+ int remaining = startupMessages.size();
+ if (remaining == 0) {
+ // While holding the lock, flip the bit so that
+ // no more events get put into startupMessages, and
+ // notify all waiters to proceed.
+ processingEvents = true;
+ startupLock.notifyAll();
+ break; // ...and we're done.
+ }
+ if (logger.isDebugEnabled()) {
+ logger.debug("Membership: {} remaining startup message(s)", remaining);
+ }
+ ev = (StartupEvent)startupMessages.removeFirst();
+ } // startupLock
+ try {
+ processStartupEvent(ev);
+ }
+ catch (VirtualMachineError err) {
+ SystemFailure.initiateFailure(err);
+ // If this ever returns, rethrow the error. We're poisoned
+ // now, so don't let this thread continue.
+ throw err;
+ }
+ catch (Throwable t) {
+ // Whenever you catch Error or Throwable, you must also
+ // catch VirtualMachineError (see above). However, there is
+ // _still_ a possibility that you are dealing with a cascading
+ // error condition, so you also need to check to see if the JVM
+ // is still usable:
+ SystemFailure.checkFailure();
+ logger.warn(LocalizedMessage.create(LocalizedStrings.GroupMembershipService_MEMBERSHIP_ERROR_HANDLING_STARTUP_EVENT), t);
+ }
+
+ } // for
+ if (logger.isDebugEnabled())
+ logger.debug("Membership: finished processing startup events.");
+ } // startupMutex
+ }
+
+
+ public void waitForEventProcessing() throws InterruptedException {
+ // First check outside of a synchronized block. Cheaper and sufficient.
+ if (Thread.interrupted()) throw new InterruptedException();
+ if (processingEvents)
+ return;
+ if (logger.isDebugEnabled()) {
+ logger.debug("Membership: waiting until the system is ready for events");
+ }
+ for (;;) {
+ directChannel.getCancelCriterion().checkCancelInProgress(null);
+ synchronized (startupLock) {
+ // Now check using a memory fence and synchronization.
+ if (processingEvents)
+ break;
+ boolean interrupted = Thread.interrupted();
+ try {
+ startupLock.wait();
+ }
+ catch (InterruptedException e) {
+ interrupted = true;
+ directChannel.getCancelCriterion().checkCancelInProgress(e);
+ }
+ finally {
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ } // synchronized
+ } // for
+ if (logger.isDebugEnabled()) {
+ logger.debug("Membership: continuing");
+ }
+ }
+
+ /**
+ * for testing we need to validate the startup event list
+ */
+ public List
+ * The lead member is the eldest member with partition detection enabled.
+ * If no members have partition detection enabled, there will be no
+ * lead member and this method will return null.
+ * @return the lead member associated with the latest view
+ */
+ public DistributedMember getLeadMember() {
+ latestViewLock.readLock().lock();
+ try {
+ return latestView == null? null : latestView.getLeadMember();
+ } finally {
+ latestViewLock.readLock().unlock();
+ }
+ }
+
+ protected boolean isJoining() {
+ return this.isJoining;
+ }
+
+ /**
+ * test hook
+ * @return the current membership view coordinator
+ */
+ public DistributedMember getCoordinator() {
+ // note - we go straight to JoinLeave because the
+ // DistributionManager queues view changes in a serial executor, where
+ // they're asynchronously installed. The DS may still see the old coordinator
+ latestViewLock.readLock().lock();
+ try {
+ return latestView == null? null : latestView.getCoordinator();
+ } finally {
+ latestViewLock.readLock().unlock();
+ }
+ }
+
+ public boolean memberExists(DistributedMember m) {
+ latestViewLock.readLock().lock();
+ NetView v = latestView;
+ latestViewLock.readLock().unlock();
+ return v.getMembers().contains(m);
+ }
+
+ /**
+ * Returns the identity associated with this member. WARNING: this value will
+ * be returned after the channel is closed, but in that case it is good for
+ * logging purposes only. :-)
+ */
+ public InternalDistributedMember getLocalMember()
+ {
+ return address;
+ }
+
+ public Services getServices() {
+ return services;
+ }
+
+ public void postConnect() {
+ }
+
+ /**
+ * @see SystemFailure#loadEmergencyClasses()
+ /**
+ * break any potential circularity in {@link #loadEmergencyClasses()}
+ */
+ private static volatile boolean emergencyClassesLoaded = false;
+
+ /**
+ * inhibits logging of ForcedDisconnectException to keep dunit logs clean
+ * while testing this feature
+ */
+ protected static volatile boolean inhibitForceDisconnectLogging;
+
+ /**
+ * Ensure that the critical classes from components
+ * get loaded.
+ *
+ * @see SystemFailure#loadEmergencyClasses()
+ */
+ public static void loadEmergencyClasses() {
+ if (emergencyClassesLoaded) return;
+ emergencyClassesLoaded = true;
+ DirectChannel.loadEmergencyClasses();
+ GMSJoinLeave.loadEmergencyClasses();
+ GMSHealthMonitor.loadEmergencyClasses();
+ }
+ /**
+ * Close the receiver, avoiding all potential deadlocks and
+ * eschewing any attempts at being graceful.
+ *
+ * @see SystemFailure#emergencyClose()
+ */
+ public void emergencyClose() {
+ final boolean DEBUG = SystemFailure.TRACE_CLOSE;
+
+ setShutdown();
+
+ // We can't call close() because they will allocate objects. Attempt
+ // a surgical strike and touch the important protocols.
+
+ // MOST important, kill the FD protocols...
+ services.emergencyClose();
+
+ // Close the TCPConduit sockets...
+ if (directChannel != null) {
+ if (DEBUG) {
+ System.err.println("DEBUG: emergency close of DirectChannel");
+ }
+ directChannel.emergencyClose();
+ }
+
+ if (DEBUG) {
+ System.err.println("DEBUG: done closing GroupMembershipService");
+ }
+ }
+
+
+ /**
+ * in order to avoid split-brain occurring when a member is shutting down due to
+ * race conditions in view management we add it as a shutdown member when we receive
+ * a shutdown message. This is not the same as a SHUNNED member.
+ */
+ public void shutdownMessageReceived(InternalDistributedMember id, String reason) {
+ if (logger.isDebugEnabled()) {
+ logger.debug("Membership: recording shutdown status of {}", id);
+ }
+ synchronized(this.shutdownMembers) {
+ this.shutdownMembers.put(id, id);
+ services.getHealthMonitor().memberShutdown(id, reason);
+ services.getJoinLeave().memberShutdown(id, reason);
+ }
+ }
+
+ /**
+ * returns true if a shutdown message has been received from the given address but
+ * that member is still in the membership view or is a surprise member.
+ */
+ public boolean isShuttingDown(InternalDistributedMember mbr) {
+ synchronized(shutdownMembers) {
+ return shutdownMembers.containsKey(mbr);
+ }
+ }
+
+
+ public void shutdown() {
+ setShutdown();
+ services.stop();
+ }
+
+ @Override
+ public void stop() {
+
+ // [bruce] Do not null out the channel w/o adding appropriate synchronization
+
+ logger.debug("MembershipManager closing");
+
+ if (directChannel != null) {
+ directChannel.disconnect(null);
+
+ if (address != null) {
+ // Make sure that channel information is consistent
+ // Probably not important in this particular case, but just
+ // to be consistent...
+ latestViewLock.writeLock().lock();
+ try {
+ destroyMember(address, false, "orderly shutdown");
+ } finally {
+ latestViewLock.writeLock().unlock();
+ }
+ }
+ }
+
+ if (cleanupTimer != null) {
+ cleanupTimer.cancel();
+ }
+
+ if (logger.isDebugEnabled()) {
+ logger.debug("Membership: channel closed");
+ }
+ }
+
+ public void uncleanShutdown(String reason, final Exception e) {
+ inhibitForcedDisconnectLogging(false);
+
+ if (services.getShutdownCause() == null) {
+ services.setShutdownCause(e);
+ }
+
+ if (this.directChannel != null) {
+ this.directChannel.disconnect(e);
+ }
+
+ // first shut down communication so we don't do any more harm to other
+ // members
+ services.emergencyClose();
+
+ if (e != null) {
+ try {
+ if (membershipTestHooks != null) {
+ List l = membershipTestHooks;
+ for (Iterator it=l.iterator(); it.hasNext(); ) {
+ MembershipTestHook dml = (MembershipTestHook)it.next();
+ dml.beforeMembershipFailure(reason, e);
+ }
+ }
+ listener.membershipFailure(reason, e);
+ if (membershipTestHooks != null) {
+ List l = membershipTestHooks;
+ for (Iterator it=l.iterator(); it.hasNext(); ) {
+ MembershipTestHook dml = (MembershipTestHook)it.next();
+ dml.afterMembershipFailure(reason, e);
+ }
+ }
+ }
+ catch (RuntimeException re) {
+ logger.warn(LocalizedMessage.create(LocalizedStrings.GroupMembershipService_EXCEPTION_CAUGHT_WHILE_SHUTTING_DOWN), re);
+ }
+ }
+ }
+
+ /** generate XML for the cache before shutting down due to forced disconnect */
+ public void saveCacheXmlForReconnect() {
+ // there are two versions of this method so it can be unit-tested
+ boolean sharedConfigEnabled = services.getConfig().getDistributionConfig().getUseSharedConfiguration();
+ saveCacheXmlForReconnect(sharedConfigEnabled);
+ }
+
+ /** generate XML from the cache before shutting down due to forced disconnect */
+ public void saveCacheXmlForReconnect(boolean sharedConfigEnabled) {
+ // first save the current cache description so reconnect can rebuild the cache
+ GemFireCacheImpl cache = GemFireCacheImpl.getInstance();
+ if (cache != null && (cache instanceof Cache)) {
+ if (!Boolean.getBoolean("gemfire.autoReconnect-useCacheXMLFile")
+ && !cache.isSqlfSystem() && !sharedConfigEnabled) {
+ try {
+ logger.info("generating XML to rebuild the cache after reconnect completes");
+ StringPrintWriter pw = new StringPrintWriter();
+ CacheXmlGenerator.generate((Cache)cache, pw, true, false);
+ String cacheXML = pw.toString();
+ cache.getCacheConfig().setCacheXMLDescription(cacheXML);
+ logger.info("XML generation completed: {}", cacheXML);
+ } catch (CancelException e) {
+ logger.info(LocalizedMessage.create(LocalizedStrings.GroupMembershipService_PROBLEM_GENERATING_CACHE_XML), e);
+ }
+ } else if (sharedConfigEnabled && !cache.getCacheServers().isEmpty()) {
+ // we need to retain a cache-server description if this JVM was started by gfsh
+ List
+ * Unlike isShunned, this method will not cause expiry of a surprise member.
+ * That must be done during view processing.
+ *
+ * Like isShunned, this method holds the view lock while executing
+ *
+ * Concurrency: protected by {@link #latestViewLock} ReentrantReadWriteLock
+ *
+ * @param m the member in question
+ * @return true if the given member is a surprise member
+ */
+ public boolean isSurpriseMember(DistributedMember m) {
+ latestViewLock.readLock().lock();
+ try {
+ if (surpriseMembers.containsKey(m)) {
+ long birthTime = ((Long)surpriseMembers.get(m)).longValue();
+ long now = System.currentTimeMillis();
+ return (birthTime >= (now - this.surpriseMemberTimeout));
+ }
+ return false;
+ } finally {
+ latestViewLock.readLock().unlock();
+ }
+ }
+
+ /**
+ * for testing we need to be able to inject surprise members into
+ * the view to ensure that sunsetting works properly
+ * @param m the member ID to add
+ * @param birthTime the millisecond clock time that the member was first seen
+ */
+ public void addSurpriseMemberForTesting(DistributedMember m, long birthTime) {
+ if (logger.isDebugEnabled()) {
+ logger.debug("test hook is adding surprise member {} birthTime={}", m, birthTime);
+ }
+ latestViewLock.writeLock().lock();
+ try {
+ surpriseMembers.put((InternalDistributedMember)m, Long.valueOf(birthTime));
+ } finally {
+ latestViewLock.writeLock().unlock();
+ }
+ }
+
+ /**
+ * returns the surpriseMemberTimeout interval, in milliseconds
+ */
+ public int getSurpriseMemberTimeout() {
+ return this.surpriseMemberTimeout;
+ }
+
+ private boolean endShun(DistributedMember m) {
+ boolean wasShunned = (shunnedMembers.remove(m) != null);
+ shunnedAndWarnedMembers.remove(m);
+ return wasShunned;
+ }
+
+ /**
+ * Add the given member to the shunned list. Also, purge any shunned
+ * members that are really really old.
+ *
+ * Must be called with {@link #latestViewLock} held and
+ * the view stable.
+ *
+ * @param m the member to add
+ */
+ protected void addShunnedMember(InternalDistributedMember m) {
+ long deathTime = System.currentTimeMillis() - SHUNNED_SUNSET * 1000;
+
+ surpriseMembers.remove(m); // for safety
+
+ // Update the shunned set.
+ if (!isShunned(m)) {
+ shunnedMembers.put(m, Long.valueOf(System.currentTimeMillis()));
+ }
+
+ // Remove really really old shunned members.
+ // First, make a copy of the old set. New arrivals _a priori_ don't matter,
+ // and we're going to be updating the list so we don't want to disturb
+ // the iterator.
+ Set oldMembers = new HashSet(shunnedMembers.entrySet());
+
+ Set removedMembers = new HashSet();
+
+ Iterator it = oldMembers.iterator();
+ while (it.hasNext()) {
+ Map.Entry e = (Map.Entry)it.next();
+
+ // Key is the member. Value is the time to remove it.
+ long ll = ((Long)e.getValue()).longValue();
+ if (ll >= deathTime) {
+ continue; // too new.
+ }
+
+ InternalDistributedMember mm = (InternalDistributedMember)e.getKey();
+
+ if (latestView.contains(mm)) {
+ // Fault tolerance: a shunned member can conceivably linger but never
+ // disconnect.
+ //
+ // We may not delete it at the time that we shun it because the view
+ // isn't necessarily stable. (Note that a well-behaved cache member
+ // will depart on its own accord, but we force the issue here.)
+ destroyMember(mm, true, "shunned but never disconnected");
+ }
+ if (logger.isDebugEnabled()) {
+ logger.debug("Membership: finally removed shunned member entry <{}>", mm);
+ }
+
+ removedMembers.add(mm);
+ }
+
+ // removed timed-out entries from the shunned-members collections
+ if (removedMembers.size() > 0) {
+ it = removedMembers.iterator();
+ while (it.hasNext()) {
+ InternalDistributedMember idm = (InternalDistributedMember)it.next();
+ endShun(idm);
+ }
+ }
+ }
+
+
+ /**
+ * Retrieve thread-local data for transport to another thread in hydra
+ */
+ public Object getThreadLocalData() {
+ Map result = new HashMap();
+ return result;
+ }
+
+ /**
+ * for testing verification purposes, this return the port for the
+ * direct channel, or zero if there is no direct
+ * channel
+ */
+ public int getDirectChannelPort() {
+ return directChannel == null? 0 : directChannel.getPort();
+ }
+
+ /**
+ * for mock testing this allows insertion of a DirectChannel mock
+ */
+ protected void setDirectChannel(DirectChannel dc) {
+ this.directChannel = dc;
+ this.tcpDisabled = false;
+ }
+
+ /* non-thread-owned serial channels and high priority channels are not
+ * included
+ */
+ public Map getMessageState(DistributedMember member, boolean includeMulticast) {
+ Map result = new HashMap();
+ DirectChannel dc = directChannel;
+ if (dc != null) {
+ dc.getChannelStates(member, result);
+ }
+ services.getMessenger().getMessageState((InternalDistributedMember)member, result, includeMulticast);
+ return result;
+ }
+
+ public void waitForMessageState(DistributedMember otherMember, Map state)
+ throws InterruptedException
+ {
+ if (Thread.interrupted()) throw new InterruptedException();
+ DirectChannel dc = directChannel;
+ if (dc != null) {
+ dc.waitForChannelState(otherMember, state);
+ }
+ services.getMessenger().waitForMessageState((InternalDistributedMember)otherMember, state);
+ }
+
+ /*
+ * (non-Javadoc)
+ * MembershipManager method: wait for the given member to be gone. Throws TimeoutException if
+ * the wait goes too long
+ * @see com.gemstone.gemfire.distributed.internal.membership.MembershipManager#waitForDeparture(com.gemstone.gemfire.distributed.DistributedMember)
+ */
+ public boolean waitForDeparture(DistributedMember mbr) throws TimeoutException, InterruptedException {
+ if (Thread.interrupted()) throw new InterruptedException();
+ boolean result = false;
+ DirectChannel dc = directChannel;
+ InternalDistributedMember idm = (InternalDistributedMember)mbr;
+ int memberTimeout = this.services.getConfig().getDistributionConfig().getMemberTimeout();
+ long pauseTime = (memberTimeout < 1000) ? 100 : memberTimeout / 10;
+ boolean wait;
+ int numWaits = 0;
+ do {
+ wait = false;
+ if (dc != null) {
+ if (dc.hasReceiversFor(idm)) {
+ wait = true;
+ }
+ if (wait && logger.isDebugEnabled()) {
+ logger.info("waiting for receivers for {} to shut down", mbr);
+ }
+ }
+ if (!wait) {
+ latestViewLock.readLock().lock();
+ try {
+ wait = this.latestView.contains(idm);
+ } finally {
+ latestViewLock.readLock().unlock();
+ }
+ if (wait && logger.isDebugEnabled()) {
+ logger.debug("waiting for {} to leave the membership view", mbr);
+ }
+ }
+ if (!wait) {
+ // run a message through the member's serial execution queue to ensure that all of its
+ // current messages have been processed
+ OverflowQueueWithDMStats serialQueue = listener.getDM().getSerialQueue(idm);
+ if (serialQueue != null) {
+ final boolean done[] = new boolean[1];
+ final FlushingMessage msg = new FlushingMessage(done);
+ serialQueue.add(new SizeableRunnable(100) {
+ public void run() {
+ msg.invoke();
+ }
+ public String toString() {
+ return "Processing fake message";
+ }
+ });
+ synchronized(done) {
+ while (done[0] == false) {
+ done.wait(10);
+ }
+ result = true;
+ }
+ }
+ }
+ if (wait) {
+ numWaits++;
+ if (numWaits > 40) {
+ // waited over 4 * memberTimeout ms. Give up at this point
+ throw new TimeoutException("waited too long for " + idm + " to be removed");
+ }
+ Thread.sleep(pauseTime);
+ }
+ } while (wait && (dc != null && dc.isOpen())
+ && services.getCancelCriterion().cancelInProgress()==null );
+ if (logger.isDebugEnabled()) {
+ logger.debug("operations for {} should all be in the cache at this point", mbr);
+ }
+ return result;
+ }
+
+
+ // TODO remove this overly complex method and replace its use with
+ // waitForViewChange using the remote member's view ID
+ public boolean waitForMembershipCheck(InternalDistributedMember remoteId) {
+ boolean foundRemoteId = false;
+ CountDownLatch currentLatch = null;
+ // ARB: preconditions
+ // remoteId != null
+ latestViewLock.writeLock().lock();
+ try {
+ if (latestView == null) {
+ // Not sure how this would happen, but see bug 38460.
+ // No view?? Not found!
+ }
+ else if (latestView.contains(remoteId)) {
+ // ARB: check if remoteId is already in membership view.
+ // If not, then create a latch if needed and wait for the latch to open.
+ foundRemoteId = true;
+ }
+ else if ((currentLatch = (CountDownLatch)this.memberLatch.get(remoteId)) == null) {
+ currentLatch = new CountDownLatch(1);
+ this.memberLatch.put(remoteId, currentLatch);
+ }
+ } finally {
+ latestViewLock.writeLock().unlock();
+ }
+
+ if (!foundRemoteId) {
+ try {
+ if (currentLatch.await(membershipCheckTimeout, TimeUnit.MILLISECONDS)) {
+ foundRemoteId = true;
+ // @todo
+ // ARB: remove latch from memberLatch map if this is the last thread waiting on latch.
+ }
+ }
+ catch (InterruptedException ex) {
+ // ARB: latch attempt was interrupted.
+ Thread.currentThread().interrupt();
+ logger.warn(LocalizedMessage.create(
+ LocalizedStrings.GroupMembershipService_THE_MEMBERSHIP_CHECK_WAS_TERMINATED_WITH_AN_EXCEPTION));
+ }
+ }
+
+ // ARB: postconditions
+ // (foundRemoteId == true) ==> (currentLatch is non-null ==> currentLatch is open)
+ return foundRemoteId;
+ }
+
+ /* returns the cause of shutdown, if known */
+ public Throwable getShutdownCause() {
+ return services.getShutdownCause();
+ }
+
+// @Override
+// public void membershipFailure(String reason, Exception e) {
+// try {
+// if (this.membershipTestHooks != null) {
+// List l = this.membershipTestHooks;
+// for (Iterator it=l.iterator(); it.hasNext(); ) {
+// MembershipTestHook dml = (MembershipTestHook)it.next();
+// dml.beforeMembershipFailure(reason, e);
+// }
+// }
+// listener.membershipFailure(reason, e);
+// if (this.membershipTestHooks != null) {
+// List l = this.membershipTestHooks;
+// for (Iterator it=l.iterator(); it.hasNext(); ) {
+// MembershipTestHook dml = (MembershipTestHook)it.next();
+// dml.afterMembershipFailure(reason, e);
+// }
+// }
+// }
+// catch (RuntimeException re) {
+// logger.warn(LocalizedMessage.create(LocalizedStrings.GroupMembershipService_EXCEPTION_CAUGHT_WHILE_SHUTTING_DOWN), re);
+// }
+// }
+
+ public void registerTestHook(MembershipTestHook mth) {
+ // lock for additions to avoid races during startup
+ latestViewLock.writeLock().lock();
+ try {
+ if (this.membershipTestHooks == null) {
+ this.membershipTestHooks = Collections.singletonList(mth);
+ }
+ else {
+ List l = new ArrayList(this.membershipTestHooks);
+ l.add(mth);
+ this.membershipTestHooks = l;
+ }
+ } finally {
+ latestViewLock.writeLock().unlock();
+ }
+ }
+
+ public void unregisterTestHook(MembershipTestHook mth) {
+ latestViewLock.writeLock().lock();
+ try {
+ if (this.membershipTestHooks != null) {
+ if (this.membershipTestHooks.size() == 1) {
+ this.membershipTestHooks = null;
+ }
+ else {
+ List l = new ArrayList(this.membershipTestHooks);
+ l.remove(mth);
+ }
+ }
+ } finally {
+ latestViewLock.writeLock().unlock();
+ }
+ }
+
+ boolean beingSick;
+ boolean playingDead;
+
+ /**
+ * Test hook - be a sick member
+ */
+ public synchronized void beSick() {
+ if (!beingSick) {
+ beingSick = true;
+ logger.info("GroupMembershipService.beSick invoked for {} - simulating sickness", this.address);
+ services.getJoinLeave().beSick();
+ services.getHealthMonitor().beSick();
+ if (directChannel != null) {
+ directChannel.beSick();
+ }
+ }
+ }
+
+ /**
+ * Test hook - don't answer "are you alive" requests
+ */
+ public synchronized void playDead() {
+ if (!playingDead) {
+ playingDead = true;
+ logger.info("GroupMembershipService.playDead invoked for {}", this.address);
+ services.getJoinLeave().playDead();
+ services.getHealthMonitor().playDead();
+ services.getMessenger().playDead();
+ }
+ }
+
+ /**
+ * Test hook - recover health
+ */
+ public synchronized void beHealthy() {
+ if (beingSick || playingDead) {
+ beingSick = false;
+ playingDead = false;
+ logger.info("GroupMembershipService.beHealthy invoked for {} - recovering health now", this.address);
+ if (directChannel != null) {
+ directChannel.beHealthy();
+ }
+ services.getJoinLeave().beHealthy();
+ services.getHealthMonitor().beHealthy();
+ services.getMessenger().beHealthy();
+ }
+ }
+
+ /**
+ * Test hook
+ */
+ public boolean isBeingSick() {
+ return this.beingSick;
+ }
+
+ /**
+ * Test hook - inhibit ForcedDisconnectException logging to keep dunit logs clean
+ * @param b
+ */
+ public static void inhibitForcedDisconnectLogging(boolean b) {
+ inhibitForceDisconnectLogging = true;
+ }
+
+ /** this is a fake message class that is used to flush the serial execution queue */
+ static class FlushingMessage extends DistributionMessage {
+ boolean[] done;
+ FlushingMessage(boolean[] done) {
+ this.done = done;
+ }
+ public void invoke() {
+ synchronized(done) {
+ done[0] = true;
+ done.notify();
+ }
+ }
+ protected void process(DistributionManager dm) {
+ // not used
+ }
+ public int getDSFID() {
+ return 0;
+ }
+ public int getProcessorType() {
+ return DistributionManager.SERIAL_EXECUTOR;
+ }
+ }
+
+ /**
+ * Sets cache time offset in {@link DistributionManager}.
+ *
+ * @param coord
+ * @param timeOffset
+ * @see InternalDistributedSystem#getClock()
+ * @see DSClock#setCacheTimeOffset(DistributedMember, long, boolean)
+ */
+ public void setCacheTimeOffset(InternalDistributedMember coord, long timeOffset, boolean isJoin) {
+ if (this.listener != null) {
+ DistributionManager dm = this.listener.getDM();
+ dm.getSystem().getClock().setCacheTimeOffset(coord, timeOffset, isJoin);
+ }
+ }
+
+ @Override
+ public void stopped() {
+ }
+
+ @Override
+ public void installView(NetView v) {
+ if (latestViewId < 0 && !isConnected()) {
+ if (this.directChannel != null) {
+ this.directChannel.setMembershipSize(v.getMembers().size());
+ }
+ latestViewId = v.getViewId();
+ latestView = v;
+ logger.debug("MembershipManager: initial view is {}", latestView);
+ } else {
+ handleOrDeferViewEvent(v);
+ }
+ }
+
+ @Override
+ public Set
+ * Command line arguments are
+ * Both the MigrationClient and MigrationServer must be configured to have
+ * the appropriate domain classes in their CLASSPATH, or errors will be
+ * encountered during deserialization.
+ *
+ * Details of the transfers can be viewed by setting the system property
+ * Migration.VERBOSE=true.
+ *
+ * For example,
+ *
+ * Where the cacheDescription.xml file might look like this:
+ *
+ * The client is then run with a different cache description having different
+ * disk-dirs to hold the migrated information.
+ *
+ * @since 6.0.1
+ */
+public class MigrationServer {
+ final static boolean VERBOSE = Boolean.getBoolean("Migration.VERBOSE");
+
+ final static int VERSION = 551; // version for backward communications compatibility
+
+ protected static final int CODE_ERROR = 0;
+ protected static final int CODE_ENTRY = 1; /* serialized key, serialized value */
+ protected static final int CODE_COMPLETED = 2;
+
+ public static void main(String[] args) throws Exception {
+ int argIdx = 0;
+ String cacheXmlFileName = "cache.xml";
+ String bindAddressName = null;
+ int listenPort = 10533;
+
+ if (args.length > 0) {
+ cacheXmlFileName = args[argIdx++];
+ } else {
+ System.err.println("MigrationServer cache-xml-file [server-address] [server-port]");
+ }
+ if (args.length > argIdx) {
+ listenPort = Integer.parseInt(args[argIdx++]);
+ }
+ if (args.length > argIdx) {
+ bindAddressName = args[argIdx++];
+ }
+
+ MigrationServer instance = null;
+ try {
+ instance = new MigrationServer(cacheXmlFileName, bindAddressName, listenPort);
+ } catch (IllegalArgumentException e) {
+ System.err.println(e.getMessage());
+ System.exit(1);
+ }
+ instance.createDistributedSystem();
+ instance.createCache();
+ instance.serve();
+ }
+
+
+ private InetAddress bindAddress;
+ private int listenPort;
+ private ServerSocket serverSocket;
+ private DistributedSystem distributedSystem;
+ private File cacheXmlFile;
+ private Cache cache;
+
+
+ /**
+ * Create a MigrationServer to be used with a DistributedSystem and Cache
+ * that are created using GemFire APIs
+ * @param bindAddressName the NIC to bind to, or null to use all interfaces
+ * @param listenPort the port to listen on
+ */
+ public MigrationServer(String bindAddressName, int listenPort) {
+ this.listenPort = listenPort;
+ if (bindAddressName != null) {
+ if (!isLocalHost(bindAddressName)) {
+ throw new IllegalArgumentException("Error - bind address is not an address of this machine: '" + bindAddressName + "'");
+ }
+ try {
+ this.bindAddress = InetAddress.getByName(bindAddressName);
+ } catch (IOException e) {
+ throw new IllegalArgumentException("Error - bind address cannot be resolved: '" + bindAddressName + "'");
+ }
+ }
+ try {
+ if (this.bindAddress != null) {
+ this.serverSocket = new ServerSocket();
+ SocketAddress addr = new InetSocketAddress(this.bindAddress, listenPort);
+ this.serverSocket.bind(addr);
+ } else {
+ this.serverSocket = new ServerSocket(listenPort);
+ }
+ if (VERBOSE) {
+ System.out.println("created server socket " + serverSocket);
+ }
+ } catch (IOException e) {
+ throw new IllegalArgumentException("Port is already in use", e);
+ }
+ }
+
+ /**
+ * this is for use by main()
+ *
+ * @param cacheXmlFileName the name of the xml file describing the cache, or null
+ * @param bindAddressName the name of the NIC to bind to, or null
+ * @param listenPort the port to listen on (must not be zero)
+ */
+ private MigrationServer(String cacheXmlFileName, String bindAddressName, int listenPort) {
+ this(bindAddressName, listenPort);
+ this.cacheXmlFile = new File(cacheXmlFileName);
+ if (!this.cacheXmlFile.exists()) {
+ // in 6.x this should be localizable
+ System.err.println("Warning - file not found in local directory: '" + cacheXmlFileName + "'");
+ }
+ }
+
+ /**
+ * Create a distributed system. If this method is not invoked before running
+ * the MigrationServer, an existing distributed system must exist for the
+ * server to use.
+ *
+ * @throws Exception if there are any problems
+ */
+ private void createDistributedSystem() throws Exception {
+ Properties dsProps = new Properties();
+ // if no discovery information has been explicitly given, use a loner ds
+ if (System.getProperty("gemfire." + DistributionConfig.MCAST_PORT_NAME) == null
+ && System.getProperty("gemfire." + DistributionConfig.LOCATORS_NAME) == null) {
+ dsProps.put(DistributionConfig.MCAST_PORT_NAME, "0");
+ }
+ dsProps.put(DistributionConfig.LOG_FILE_NAME, "migrationServer.log");
+ if (this.cacheXmlFile != null) {
+ dsProps.put(DistributionConfig.CACHE_XML_FILE_NAME, this.cacheXmlFile.getName());
+ }
+ this.distributedSystem = DistributedSystem.connect(dsProps);
+ if (VERBOSE) {
+ System.out.println("created distributed system " + this.distributedSystem);
+ }
+ }
+
+
+ /**
+ * create the cache to be used by this migration server
+ * @throws Exception if there are any problems
+ */
+ private void createCache() throws Exception {
+ if (this.distributedSystem == null) {
+ this.distributedSystem = InternalDistributedSystem.getConnectedInstance();
+ }
+ this.cache = CacheFactory.create(this.distributedSystem);
+ if (VERBOSE) {
+ System.out.println("created cache " + this.cache);
+ }
+ }
+
+
+ /**
+ * This locates the distributed system and cache, if they have not been
+ * created by this server, and then listens for requests from MigrationClient
+ * processes.
+ * @throws IllegalStateException if an attempt is made to reuse a server that has been stopped
+ */
+ public void serve() throws Exception {
+ if (this.serverSocket == null || this.serverSocket.isClosed()) {
+ throw new IllegalStateException("This server has been closed and cannot be reused");
+ }
+ try {
+ if (this.distributedSystem == null) {
+ this.distributedSystem = InternalDistributedSystem.getConnectedInstance();
+ }
+ if (this.cache == null) {
+ this.cache = GemFireCacheImpl.getInstance();
+ }
+ if (this.bindAddress != null) {
+ System.out.println("Migration server on port " + this.listenPort +
+ " bound to " + this.bindAddress + " is ready for client requets");
+ } else {
+ System.out.println("Migration server on port " + this.listenPort +
+ " is ready for client requests");
+ }
+ for (;;) {
+ if (Thread.interrupted() || this.serverSocket.isClosed()) {
+ return;
+ }
+ Socket clientSocket;
+ try {
+ clientSocket = this.serverSocket.accept();
+ } catch (java.net.SocketException e) {
+ return;
+ }
+ (new RequestHandler(clientSocket)).serveClientRequest();
+ }
+ } finally {
+ System.out.println("Closing migration server");
+ try {
+ this.serverSocket.close();
+ } catch (Exception e) {
+ this.serverSocket = null;
+ }
+ }
+ }
+
+ /**
+ * this causes the migration server to stop serving after it finishes dispatching
+ * any in-process requests
+ * @throws IOException if there is a problem closing the server socket
+ */
+ public void stop() throws IOException {
+ if (this.serverSocket != null && !this.serverSocket.isClosed()) {
+ this.serverSocket.close();
+ }
+ }
+
+ /**
+ * get the cache being used by this migration server
+ * @return the cache, or null if a cache has not yet been associated with this server
+ */
+ public Cache getCache() {
+ return this.cache;
+ }
+
+ /**
+ * get the distributed system being used by this migration server
+ * @return the distributed system, or null if a system has not yet been associated with this server
+ */
+ public DistributedSystem getDistributedSystem() {
+ return this.distributedSystem;
+ }
+
+
+
+
+
+
+ // copied from 6.0 SocketCreator
+ public static boolean isLocalHost(Object host) {
+ if (host instanceof InetAddress) {
+ if (InetAddressUtil.LOCALHOST.equals(host)) {
+ return true;
+ }
+ else {
+ try {
+ Enumeration en=NetworkInterface.getNetworkInterfaces();
+ while(en.hasMoreElements()) {
+ NetworkInterface i=(NetworkInterface)en.nextElement();
+ for(Enumeration en2=i.getInetAddresses(); en2.hasMoreElements();) {
+ InetAddress addr=(InetAddress)en2.nextElement();
+ if (host.equals(addr)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+ catch (SocketException e) {
+ throw new IllegalArgumentException(LocalizedStrings.InetAddressUtil_UNABLE_TO_QUERY_NETWORK_INTERFACE.toLocalizedString(), e);
+ }
+ }
+ }
+ else {
+ return isLocalHost(toInetAddress(host.toString()));
+ }
+ }
+
+ // copied from 6.0 SocketCreator
+ public static InetAddress toInetAddress(String host) {
+ if (host == null || host.length() == 0) {
+ return null;
+ }
+ try {
+ if (host.indexOf("/") > -1) {
+ return InetAddress.getByName(host.substring(host.indexOf("/") + 1));
+ }
+ else {
+ return InetAddress.getByName(host);
+ }
+ } catch (java.net.UnknownHostException e) {
+ throw new IllegalArgumentException(e.getMessage());
+ }
+ }
+
+
+
+
+
+ // R E Q U E S T H A N D L E R
+
+
+ class RequestHandler implements Runnable {
+ Socket clientSocket;
+ DataInputStream dis;
+ DataOutputStream dos;
+
+ RequestHandler(Socket clientSocket) throws IOException {
+ this.clientSocket = clientSocket;
+ dos = new DataOutputStream(this.clientSocket.getOutputStream());
+ dis = new DataInputStream(this.clientSocket.getInputStream());
+ }
+
+
+ // for now this is a blocking operation - multithread later if necessary
+ void serveClientRequest() {
+ try {
+ run();
+ }
+ finally {
+ if (!this.clientSocket.isClosed()) {
+ try {
+ this.clientSocket.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ public void run() {
+ try {
+ // first exchange version information so we can communicate correctly
+ dos.writeShort(VERSION);
+ int version = dis.readShort();
+ handleRequest(version);
+ }
+ catch (IOException e) {
+ System.err.println("Trouble dispatching request: " + e.getMessage());
+ return;
+ }
+ finally {
+ try {
+ this.clientSocket.close();
+ } catch (IOException e) {
+ System.err.println("Trouble closing client socket: " + e.getMessage());
+ }
+ }
+ }
+
+ /**
+ * read and dispatch a single request on client socket
+ * @param clientVersion
+ */
+ private void handleRequest(int clientVersion) {
+ // for now we ignore the client version in the server. The client
+ // is typically of a later release than the server, and this information
+ // is given to the server in case a situation arises where it's needed
+ try {
+ ClientRequest req = ClientRequest.readRequest(this.clientSocket, dis, dos);
+ if (req != null) {
+ System.out.println("Processing " + req + " from " + this.clientSocket.getInetAddress().getHostAddress());
+ req.process(MigrationServer.this);
+ dos.flush();
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ }
+
+
+ // R E Q U E S T C L A S S E S
+
+
+
+ static abstract class ClientRequest {
+ Socket clientSocket;
+ DataInputStream dsi;
+ DataOutputStream dso;
+
+ final static int REGION_REQUEST = 1;
+
+ /**
+ * Use readRequest to create a new request object, not this constructor.
+ * Subclasses may refine this constructor to perform other initialization
+ * @param dsi socket's input stream
+ * @param dso socket's output stream
+ * @throws IOException if there are any problems reading initialization information
+ */
+ ClientRequest(Socket clientSocket, DataInputStream dsi, DataOutputStream dso) throws IOException {
+ this.clientSocket = clientSocket;
+ this.dsi = dsi;
+ this.dso = dso;
+ }
+
+ /**
+ * Read and return a request from a client
+ * @param clientSocket
+ * @param dsi socket input stream
+ * @param dso socket output stream
+ * @return the new request
+ * @throws IOException
+ */
+ static ClientRequest readRequest(Socket clientSocket, DataInputStream dsi, DataOutputStream dso) throws IOException {
+ int requestType = dsi.readShort();
+ switch (requestType) {
+ case REGION_REQUEST:
+ return new RegionRequest(clientSocket, dsi, dso);
+ }
+ String errorMessage = "Type of request is not implemented in this server";
+ dso.writeShort(CODE_ERROR);
+ dso.writeUTF(errorMessage);
+ System.err.println("Migration server received unknown type of request ("
+ + requestType + ") from " + clientSocket.getInetAddress().getHostAddress());
+ return null;
+ }
+
+ void writeErrorResponse(String message) throws IOException {
+ this.dso.writeShort(CODE_ERROR);
+ this.dso.writeUTF(message);
+ }
+
+ abstract void process(MigrationServer server) throws IOException ;
+
+ }
+
+ /**
+ * RegionRequest represents a request for the keys and values of a Region
+ * from a client.
+ */
+ static class RegionRequest extends ClientRequest {
+ String regionName;
+
+ RegionRequest(Socket clientSocket, DataInputStream dsi, DataOutputStream dso) throws IOException {
+ super(clientSocket, dsi, dso);
+ regionName = dsi.readUTF();
+ }
+
+ @Override
+ public String toString() {
+ return "request for contents of region '" + this.regionName + "'";
+ }
+
+ @Override
+ void process(MigrationServer server) throws IOException {
+ Cache cache = server.getCache();
+ Region region = null;
+ try {
+ region = cache.getRegion(regionName);
+ if (region == null) {
+ String errorMessage = "Error: region " + this.regionName + " not found in cache";
+ System.err.println(errorMessage);
+ writeErrorResponse(errorMessage);
+ }
+ } catch (IllegalArgumentException e) {
+ String errorMessage = "Error: malformed region name";
+ System.err.println(errorMessage);
+ writeErrorResponse(errorMessage);
+ }
+ try {
+ for (Iterator it = region.entrySet().iterator(); it.hasNext(); ) {
+ sendEntry((Region.Entry)it.next());
+ }
+ this.dso.writeShort(CODE_COMPLETED);
+ }
+ catch (Exception e) {
+ writeErrorResponse(e.getMessage());
+ }
+ }
+
+ private void sendEntry(Region.Entry entry) throws Exception {
+ Object key = entry.getKey();
+ Object value = entry.getValue();
+ if ( !(key instanceof Serializable) ) {
+ throw new IOException("Could not serialize entry for '" + key + "'");
+ }
+ if ( !(value instanceof Serializable) ) {
+ throw new IOException("Could not serialize entry for '" + key + "'");
+ }
+ if (VERBOSE) {
+ System.out.println("Sending " + key);
+ }
+ dso.writeShort(CODE_ENTRY);
+ (new ObjectOutputStream(clientSocket.getOutputStream())).writeObject(key);
+ (new ObjectOutputStream(clientSocket.getOutputStream())).writeObject(value);
+ }
+ }
+
+}
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionEntry.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionEntry.java
new file mode 100644
index 000000000000..327170ea529b
--- /dev/null
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionEntry.java
@@ -0,0 +1,2241 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.gemstone.gemfire.internal.cache;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.logging.log4j.Logger;
+
+import static com.gemstone.gemfire.internal.offheap.annotations.OffHeapIdentifier.ABSTRACT_REGION_ENTRY_FILL_IN_VALUE;
+import static com.gemstone.gemfire.internal.offheap.annotations.OffHeapIdentifier.ABSTRACT_REGION_ENTRY_PREPARE_VALUE_FOR_CACHE;
+
+import com.gemstone.gemfire.CancelException;
+import com.gemstone.gemfire.InvalidDeltaException;
+import com.gemstone.gemfire.SystemFailure;
+import com.gemstone.gemfire.cache.CacheWriterException;
+import com.gemstone.gemfire.cache.EntryEvent;
+import com.gemstone.gemfire.cache.EntryNotFoundException;
+import com.gemstone.gemfire.cache.Operation;
+import com.gemstone.gemfire.cache.TimeoutException;
+import com.gemstone.gemfire.cache.query.IndexMaintenanceException;
+import com.gemstone.gemfire.cache.query.QueryException;
+import com.gemstone.gemfire.cache.query.internal.index.IndexManager;
+import com.gemstone.gemfire.cache.query.internal.index.IndexProtocol;
+import com.gemstone.gemfire.cache.util.GatewayConflictHelper;
+import com.gemstone.gemfire.cache.util.GatewayConflictResolver;
+import com.gemstone.gemfire.distributed.internal.DM;
+import com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember;
+import com.gemstone.gemfire.internal.Assert;
+import com.gemstone.gemfire.internal.ByteArrayDataInput;
+import com.gemstone.gemfire.internal.HeapDataOutputStream;
+import com.gemstone.gemfire.internal.InternalDataSerializer;
+import com.gemstone.gemfire.internal.InternalStatisticsDisabledException;
+import com.gemstone.gemfire.internal.Version;
+import com.gemstone.gemfire.internal.cache.lru.LRUClockNode;
+import com.gemstone.gemfire.internal.cache.lru.NewLRUClockHand;
+import com.gemstone.gemfire.internal.cache.persistence.DiskStoreID;
+import com.gemstone.gemfire.internal.cache.versions.ConcurrentCacheModificationException;
+import com.gemstone.gemfire.internal.cache.versions.RegionVersionVector;
+import com.gemstone.gemfire.internal.cache.versions.VersionSource;
+import com.gemstone.gemfire.internal.cache.versions.VersionStamp;
+import com.gemstone.gemfire.internal.cache.versions.VersionTag;
+import com.gemstone.gemfire.internal.cache.wan.GatewaySenderEventImpl;
+import com.gemstone.gemfire.internal.i18n.LocalizedStrings;
+import com.gemstone.gemfire.internal.lang.StringUtils;
+import com.gemstone.gemfire.internal.logging.LogService;
+import com.gemstone.gemfire.internal.logging.log4j.LocalizedMessage;
+import com.gemstone.gemfire.internal.logging.log4j.LogMarker;
+import com.gemstone.gemfire.internal.offheap.Chunk;
+import com.gemstone.gemfire.internal.offheap.ChunkWithHeapForm;
+import com.gemstone.gemfire.internal.offheap.GemFireChunk;
+import com.gemstone.gemfire.internal.offheap.MemoryAllocator;
+import com.gemstone.gemfire.internal.offheap.OffHeapCachedDeserializable;
+import com.gemstone.gemfire.internal.offheap.OffHeapHelper;
+import com.gemstone.gemfire.internal.offheap.ReferenceCountHelper;
+import com.gemstone.gemfire.internal.offheap.SimpleMemoryAllocatorImpl;
+import com.gemstone.gemfire.internal.offheap.StoredObject;
+import com.gemstone.gemfire.internal.offheap.annotations.Released;
+import com.gemstone.gemfire.internal.offheap.annotations.Retained;
+import com.gemstone.gemfire.internal.offheap.annotations.Unretained;
+import com.gemstone.gemfire.internal.util.BlobHelper;
+import com.gemstone.gemfire.internal.util.Versionable;
+import com.gemstone.gemfire.internal.util.concurrent.CustomEntryConcurrentHashMap;
+import com.gemstone.gemfire.internal.util.concurrent.CustomEntryConcurrentHashMap.HashEntry;
+import com.gemstone.gemfire.pdx.PdxInstance;
+import com.gemstone.gemfire.pdx.PdxSerializable;
+import com.gemstone.gemfire.pdx.PdxSerializationException;
+import com.gemstone.gemfire.pdx.PdxSerializer;
+import com.gemstone.gemfire.pdx.internal.ConvertableToBytes;
+import com.gemstone.gemfire.pdx.internal.PdxInstanceImpl;
+
+/**
+ * Abstract implementation class of RegionEntry interface.
+ * This is the topmost implementation class so common behavior
+ * lives here.
+ *
+ * @since 3.5.1
+ *
+ *
+ */
+public abstract class AbstractRegionEntry implements RegionEntry,
+ HashEntry
+ *
+ * Note that a tag from a remote (WAN) system may be in the event. If this
+ * is the case this method will either invoke a user plugin that allows/disallows
+ * the event (and may modify the value) or it determines whether to allow
+ * or disallow the event based on timestamps and distributedSystemIDs.
+ *
+ * @throws ConcurrentCacheModificationException if the event conflicts with
+ * an event that has already been applied to the entry.
+ *
+ * @see com.gemstone.gemfire.internal.cache.RegionEntry#concurrencyCheck(com.gemstone.gemfire.cache.EntryEvent)
+ */
+ public void processVersionTag(EntryEvent cacheEvent) {
+ processVersionTag(cacheEvent, true);
+ }
+
+
+ protected void processVersionTag(EntryEvent cacheEvent, boolean conflictCheck) {
+ EntryEventImpl event = (EntryEventImpl)cacheEvent;
+ VersionTag tag = event.getVersionTag();
+ if (tag == null) {
+ return;
+ }
+
+ try {
+ if (tag.isGatewayTag()) {
+ // this may throw ConcurrentCacheModificationException or modify the event
+ if (processGatewayTag(cacheEvent)) {
+ return;
+ }
+ assert false : "processGatewayTag failure - returned false";
+ }
+
+ if (!tag.isFromOtherMember()) {
+ if (!event.getOperation().isNetSearch()) {
+ // except for netsearch, all locally-generated tags can be ignored
+ return;
+ }
+ }
+
+ final InternalDistributedMember originator = (InternalDistributedMember)event.getDistributedMember();
+ final VersionSource dmId = event.getRegion().getVersionMember();
+ LocalRegion r = event.getLocalRegion();
+ boolean eventHasDelta = event.getDeltaBytes() != null && event.getRawNewValue() == null;
+
+ VersionStamp stamp = getVersionStamp();
+ // bug #46223, an event received from a peer or a server may be from a different
+ // distributed system than the last modification made to this entry so we must
+ // perform a gateway conflict check
+ if (stamp != null && !tag.isAllowedByResolver()) {
+ int stampDsId = stamp.getDistributedSystemId();
+ int tagDsId = tag.getDistributedSystemId();
+
+ if (stampDsId != 0 && stampDsId != tagDsId && stampDsId != -1) {
+ StringBuilder verbose = null;
+ if (logger.isTraceEnabled(LogMarker.TOMBSTONE)) {
+ verbose = new StringBuilder();
+ verbose.append("processing tag for key " + getKey() + ", stamp=" + stamp.asVersionTag() + ", tag=").append(tag);
+ }
+ long stampTime = stamp.getVersionTimeStamp();
+ long tagTime = tag.getVersionTimeStamp();
+ if (stampTime > 0 && (tagTime > stampTime
+ || (tagTime == stampTime && tag.getDistributedSystemId() >= stamp.getDistributedSystemId()))) {
+ if (verbose != null) {
+ verbose.append(" - allowing event");
+ logger.trace(LogMarker.TOMBSTONE, verbose);
+ }
+ // Update the stamp with event's version information.
+ applyVersionTag(r, stamp, tag, originator);
+ return;
+ }
+
+ if (stampTime > 0) {
+ if (verbose != null) {
+ verbose.append(" - disallowing event");
+ logger.trace(LogMarker.TOMBSTONE, verbose);
+ }
+ r.getCachePerfStats().incConflatedEventsCount();
+ persistConflictingTag(r, tag);
+ throw new ConcurrentCacheModificationException("conflicting event detected");
+ }
+ }
+ }
+
+ if (r.getVersionVector() != null &&
+ r.getServerProxy() == null &&
+ (r.getDataPolicy().withPersistence() ||
+ !r.getScope().isLocal())) { // bug #45258 - perf degradation for local regions and RVV
+ VersionSource who = tag.getMemberID();
+ if (who == null) {
+ who = originator;
+ }
+ r.getVersionVector().recordVersion(who, tag);
+ }
+
+ assert !tag.isFromOtherMember() || tag.getMemberID() != null : "remote tag is missing memberID";
+
+
+ // [bruce] for a long time I had conflict checks turned off in clients when
+ // receiving a response from a server and applying it to the cache. This lowered
+ // the CPU cost of versioning but eventually had to be pulled for bug #45453
+// if (r.getServerProxy() != null && conflictCheck) {
+// // events coming from servers while a local sync is held on the entry
+// // do not require a conflict check. Conflict checks were already
+// // performed on the server and here we just consume whatever was sent back.
+// // Event.isFromServer() returns true for client-update messages and
+// // for putAll/getAll, which do not hold syncs during the server operation.
+// conflictCheck = event.isFromServer();
+// }
+// else
+
+ // [bruce] for a very long time we had conflict checks turned off for PR buckets.
+ // Bug 45669 showed a primary dying in the middle of distribution. This caused
+ // one backup bucket to have a v2. The other bucket was promoted to primary and
+ // generated a conflicting v2. We need to do the check so that if this second
+ // v2 loses to the original one in the delta-GII operation that the original v2
+ // will be the winner in both buckets.
+// if (r.isUsedForPartitionedRegionBucket()) {
+// conflictCheck = false; // primary/secondary model
+// }
+
+ // The new value in event is not from GII, even it could be tombstone
+ basicProcessVersionTag(r, tag, false, eventHasDelta, dmId, originator, conflictCheck);
+ } catch (ConcurrentCacheModificationException ex) {
+ event.isConcurrencyConflict(true);
+ throw ex;
+ }
+ }
+
+ protected final void basicProcessVersionTag(LocalRegion region, VersionTag tag, boolean isTombstoneFromGII,
+ boolean deltaCheck, VersionSource dmId, InternalDistributedMember sender, boolean checkForConflict) {
+
+ StringBuilder verbose = null;
+
+ if (tag != null) {
+ VersionStamp stamp = getVersionStamp();
+
+ if (logger.isTraceEnabled(LogMarker.TOMBSTONE)) {
+ VersionTag stampTag = stamp.asVersionTag();
+ if (stampTag.hasValidVersion() && checkForConflict) { // only be verbose here if there's a possibility we might reject the operation
+ verbose = new StringBuilder();
+ verbose.append("processing tag for key " + getKey() + ", stamp=" + stamp.asVersionTag() + ", tag=").append(tag)
+ .append(", checkForConflict=").append(checkForConflict); //.append(", current value=").append(_getValue());
+ }
+ }
+
+ if (stamp == null) {
+ throw new IllegalStateException("message contained a version tag but this region has no version storage");
+ }
+
+ boolean apply = true;
+
+ try {
+ if (checkForConflict) {
+ apply = checkForConflict(region, stamp, tag, isTombstoneFromGII, deltaCheck, dmId, sender, verbose);
+ }
+ } catch (ConcurrentCacheModificationException e) {
+ // Even if we don't apply the operation we should always retain the
+ // highest timestamp in order for WAN conflict checks to work correctly
+ // because the operation may have been sent to other systems and been
+ // applied there
+ if (!tag.isGatewayTag()
+ && stamp.getDistributedSystemId() == tag.getDistributedSystemId()
+ && tag.getVersionTimeStamp() > stamp.getVersionTimeStamp()) {
+ stamp.setVersionTimeStamp(tag.getVersionTimeStamp());
+ tag.setTimeStampApplied(true);
+ if (verbose != null) {
+ verbose.append("\nThough in conflict the tag timestamp was more recent and was recorded.");
+ }
+ }
+ throw e;
+ } finally {
+ if (verbose != null) {
+ logger.trace(LogMarker.TOMBSTONE, verbose);
+ }
+ }
+
+ if (apply) {
+ applyVersionTag(region, stamp, tag, sender);
+ }
+ }
+ }
+
+
+ private void applyVersionTag(LocalRegion region, VersionStamp stamp, VersionTag tag, InternalDistributedMember sender) {
+ // stamp.setPreviousMemberID(stamp.getMemberID());
+ VersionSource mbr = tag.getMemberID();
+ if (mbr == null) {
+ mbr = sender;
+ }
+ mbr = region.getVersionVector().getCanonicalId(mbr);
+ tag.setMemberID(mbr);
+ stamp.setVersions(tag);
+ if (tag.hasPreviousMemberID()) {
+ if (tag.getPreviousMemberID() == null) {
+ tag.setPreviousMemberID(stamp.getMemberID());
+ } else {
+ tag.setPreviousMemberID(region.getVersionVector().getCanonicalId(
+ tag.getPreviousMemberID()));
+ }
+ }
+ }
+
+ /** perform conflict checking for a stamp/tag */
+ protected boolean checkForConflict(LocalRegion region,
+ VersionStamp stamp, VersionTag tag,
+ boolean isTombstoneFromGII,
+ boolean deltaCheck, VersionSource dmId,
+ InternalDistributedMember sender, StringBuilder verbose) {
+
+ int stampVersion = stamp.getEntryVersion();
+ int tagVersion = tag.getEntryVersion();
+
+ boolean throwex = false;
+ boolean apply = false;
+
+ if (stamp.getVersionTimeStamp() != 0) { // new entries have no timestamp
+ // check for wrap-around on the version number
+ long difference = tagVersion - stampVersion;
+ if (0x10000 < difference || difference < -0x10000) {
+ if (verbose != null) {
+ verbose.append("\nversion rollover detected: tag="+tagVersion + " stamp=" + stampVersion);
+ }
+ if (difference < 0) {
+ tagVersion += 0x1000000L;
+ } else {
+ stampVersion += 0x1000000L;
+ }
+ }
+ }
+ if (verbose != null) {
+ verbose.append("\nstamp=v").append(stampVersion)
+ .append(" tag=v").append(tagVersion);
+ }
+
+ if (deltaCheck) {
+ checkForDeltaConflict(region, stampVersion, tagVersion, stamp, tag, dmId, sender, verbose);
+ }
+
+ if (stampVersion == 0 || stampVersion < tagVersion) {
+ if (verbose != null) { verbose.append(" - applying change"); }
+ apply = true;
+ } else if (stampVersion > tagVersion) {
+ if (overwritingOldTombstone(region, stamp, tag, verbose) && tag.getVersionTimeStamp() > stamp.getVersionTimeStamp()) {
+ apply = true;
+ } else {
+ // check for an incoming expired tombstone from an initial image chunk.
+ if (tagVersion > 0
+ && isExpiredTombstone(region, tag.getVersionTimeStamp(), isTombstoneFromGII)
+ && tag.getVersionTimeStamp() > stamp.getVersionTimeStamp()) {
+ // A special case to apply: when remote entry is expired tombstone, then let local vs remote with newer timestamp to win
+ if (verbose != null) { verbose.append(" - applying change in Delta GII"); }
+ apply = true;
+ } else {
+ if (verbose != null) { verbose.append(" - disallowing"); }
+ throwex= true;
+ }
+ }
+ } else {
+ if (overwritingOldTombstone(region, stamp, tag, verbose)) {
+ apply = true;
+ } else {
+ // compare member IDs
+ VersionSource stampID = stamp.getMemberID();
+ if (stampID == null) {
+ stampID = dmId;
+ }
+ VersionSource tagID = tag.getMemberID();
+ if (tagID == null) {
+ tagID = sender;
+ }
+ if (verbose != null) { verbose.append("\ncomparing IDs"); }
+ int compare = stampID.compareTo(tagID);
+ if (compare < 0) {
+ if (verbose != null) { verbose.append(" - applying change"); }
+ apply = true;
+ } else if (compare > 0) {
+ if (verbose != null) { verbose.append(" - disallowing"); }
+ throwex = true;
+ } else if (tag.isPosDup()) {
+ if (verbose != null) { verbose.append(" - disallowing duplicate marked with posdup"); }
+ throwex = true;
+ } else /* if (isTombstoneFromGII && isTombstone()) {
+ if (verbose != null) { verbose.append(" - disallowing duplicate tombstone from GII"); }
+ return false; // bug #49601 don't schedule tombstones from GII if there's already one here
+ } else */ {
+ if (verbose != null) { verbose.append(" - allowing duplicate"); }
+ }
+ }
+ }
+
+ if (!apply && throwex) {
+ region.getCachePerfStats().incConflatedEventsCount();
+ persistConflictingTag(region, tag);
+ throw new ConcurrentCacheModificationException();
+ }
+
+ return apply;
+ }
+
+ private boolean isExpiredTombstone(LocalRegion region, long timestamp, boolean isTombstone) {
+ return isTombstone && (timestamp + TombstoneService.REPLICATED_TOMBSTONE_TIMEOUT) <= region.cacheTimeMillis();
+ }
+
+ private boolean overwritingOldTombstone(LocalRegion region, VersionStamp stamp, VersionTag tag, StringBuilder verbose) {
+ // Tombstone GC does not use locking to stop operations when old tombstones
+ // are being removed. Because of this we might get an operation that was applied
+ // in another VM that has just reaped a tombstone and is now using a reset
+ // entry version number. Because of this we check the timestamp on the current
+ // local entry and see if it is old enough to have expired. If this is the case
+ // we accept the change and allow the tag to be recorded
+ long stampTime = stamp.getVersionTimeStamp();
+ if (isExpiredTombstone(region, stampTime, this.isTombstone())) {
+ // no local change since the tombstone would have timed out - accept the change
+ if (verbose != null) { verbose.append(" - accepting because local timestamp is old"); }
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ protected void persistConflictingTag(LocalRegion region, VersionTag tag) {
+ // only persist region needs to persist conflict tag
+ }
+
+ /**
+ * for an event containing a delta we must check to see if the tag's
+ * previous member id is the stamp's member id and ensure that the
+ * version is only incremented by 1. Otherwise the delta is being
+ * applied to a value that does not match the source of the delta.
+ *
+ * @throws InvalidDeltaException
+ */
+ private void checkForDeltaConflict(LocalRegion region,
+ long stampVersion, long tagVersion,
+ VersionStamp stamp, VersionTag tag,
+ VersionSource dmId, InternalDistributedMember sender,
+ StringBuilder verbose) {
+
+ if (tagVersion != stampVersion+1) {
+ if (verbose != null) {
+ verbose.append("\ndelta requires full value due to version mismatch");
+ }
+ region.getCachePerfStats().incDeltaFailedUpdates();
+ throw new InvalidDeltaException("delta cannot be applied due to version mismatch");
+
+ } else {
+ // make sure the tag was based on the value in this entry by checking the
+ // tag's previous-changer ID against this stamp's current ID
+ VersionSource stampID = stamp.getMemberID();
+ if (stampID == null) {
+ stampID = dmId;
+ }
+ VersionSource tagID = tag.getPreviousMemberID();
+ if (tagID == null) {
+ tagID = sender;
+ }
+ if (!tagID.equals(stampID)) {
+ if (verbose != null) {
+ verbose.append("\ndelta requires full value. tag.previous=")
+ .append(tagID).append(" but stamp.current=").append(stampID);
+ }
+ region.getCachePerfStats().incDeltaFailedUpdates();
+ throw new InvalidDeltaException("delta cannot be applied due to version ID mismatch");
+ }
+ }
+ }
+
+ private boolean processGatewayTag(EntryEvent cacheEvent) {
+ // Gateway tags are installed in the server-side LocalRegion cache
+ // modification methods. They do not have version numbers or distributed
+ // member IDs. Instead they only have timestamps and distributed system IDs.
+
+ // If there is a resolver plug-in, invoke it. Otherwise we use the timestamps and
+ // distributed system IDs to determine whether to allow the event to proceed.
+
+ final boolean isDebugEnabled = logger.isDebugEnabled();
+
+ if (this.isRemoved() && !this.isTombstone()) {
+ return true; // no conflict on a new entry
+ }
+ EntryEventImpl event = (EntryEventImpl)cacheEvent;
+ VersionTag tag = event.getVersionTag();
+ long stampTime = getVersionStamp().getVersionTimeStamp();
+ long tagTime = tag.getVersionTimeStamp();
+ int stampDsid = getVersionStamp().getDistributedSystemId();
+ int tagDsid = tag.getDistributedSystemId();
+ if (isDebugEnabled) {
+ logger.debug("processing gateway version information for {}. Stamp dsid={} time={} Tag dsid={} time={}",
+ event.getKey(), stampDsid, stampTime, tagDsid, tagTime);
+ }
+ if (tagTime == VersionTag.ILLEGAL_VERSION_TIMESTAMP) {
+ return true; // no timestamp received from other system - just apply it
+ }
+ if (tagDsid == stampDsid || stampDsid == -1) {
+ return true;
+ }
+ GatewayConflictResolver resolver = event.getRegion().getCache().getGatewayConflictResolver();
+ if (resolver != null) {
+ if (isDebugEnabled) {
+ logger.debug("invoking gateway conflict resolver");
+ }
+ final boolean[] disallow = new boolean[1];
+ final Object[] newValue = new Object[] { this };
+ GatewayConflictHelper helper = new GatewayConflictHelper() {
+ @Override
+ public void disallowEvent() {
+ disallow[0] = true;
+ }
+
+ @Override
+ public void changeEventValue(Object v) {
+ newValue[0] = v;
+ }
+ };
+ TimestampedEntryEventImpl timestampedEvent =
+ (TimestampedEntryEventImpl)event.getTimestampedEvent(tagDsid, stampDsid, tagTime, stampTime);
+
+ // gateway conflict resolvers will usually want to see the old value
+ if (!timestampedEvent.hasOldValue() && isRemoved()) {
+ timestampedEvent.setOldValue(getValue(timestampedEvent.getRegion())); // OFFHEAP: since isRemoved I think getValue will never be stored off heap in this case
+ }
+
+ Throwable thr = null;
+ try {
+ resolver.onEvent(timestampedEvent, helper);
+ }
+ catch (CancelException cancelled) {
+ throw cancelled;
+ }
+ catch (VirtualMachineError err) {
+ SystemFailure.initiateFailure(err);
+ // If this ever returns, rethrow the error. We're poisoned
+ // now, so don't let this thread continue.
+ throw err;
+ }
+ catch (Throwable t) {
+ // Whenever you catch Error or Throwable, you must also
+ // catch VirtualMachineError (see above). However, there is
+ // _still_ a possibility that you are dealing with a cascading
+ // error condition, so you also need to check to see if the JVM
+ // is still usable:
+ SystemFailure.checkFailure();
+ logger.error(LocalizedMessage.create(LocalizedStrings.LocalRegion_EXCEPTION_OCCURRED_IN_CONFLICTRESOLVER), t);
+ thr = t;
+ } finally {
+ timestampedEvent.release();
+ }
+
+ if (isDebugEnabled) {
+ logger.debug("done invoking resolver {}", thr);
+ }
+ if (thr == null) {
+ if (disallow[0]) {
+ if (isDebugEnabled) {
+ logger.debug("conflict resolver rejected the event for {}", event.getKey());
+ }
+ throw new ConcurrentCacheModificationException("WAN conflict resolver rejected the operation");
+ }
+
+ tag.setAllowedByResolver(true);
+
+ if (newValue[0] != this) {
+ if (isDebugEnabled) {
+ logger.debug("conflict resolver changed the value of the event for {}", event.getKey());
+ }
+ // the resolver changed the event value!
+ event.setNewValue(newValue[0]);
+ }
+ // if nothing was done then we allow the event
+ if (isDebugEnabled) {
+ logger.debug("change was allowed by conflict resolver: {}", tag);
+ }
+ return true;
+ }
+ }
+ if (isDebugEnabled) {
+ logger.debug("performing normal WAN conflict check");
+ }
+ if (tagTime > stampTime
+ || (tagTime == stampTime && tagDsid >= stampDsid)) {
+ if (isDebugEnabled) {
+ logger.debug("allowing event");
+ }
+ return true;
+ }
+ if (isDebugEnabled) {
+ logger.debug("disallowing event for " + event.getKey());
+ }
+ throw new ConcurrentCacheModificationException("conflicting WAN event detected");
+ }
+
+ static boolean isCompressible(RegionEntryContext context,Object value) {
+ return ((value != null) && (context != null) && (context.getCompressor() != null) && !Token.isInvalidOrRemoved(value));
+ }
+
+ /* subclasses supporting versions must override this */
+ public VersionStamp getVersionStamp() {
+ return null;
+ }
+
+ public boolean isValueNull() {
+ return (null == getValueAsToken());
+ }
+
+ public boolean isInvalid() {
+ return Token.isInvalid(getValueAsToken());
+ }
+
+ public boolean isDestroyed() {
+ return Token.isDestroyed(getValueAsToken());
+ }
+
+ public void setValueToNull() {
+ _setValue(null);
+ }
+
+ public boolean isInvalidOrRemoved() {
+ return Token.isInvalidOrRemoved(getValueAsToken());
+ }
+
+ /**
+ * Maximum size of a string that can be encoded as char.
+ */
+ public static final int MAX_INLINE_STRING_KEY_CHAR_ENCODING = 7;
+ /**
+ * Maximum size of a string that can be encoded as byte.
+ */
+ public static final int MAX_INLINE_STRING_KEY_BYTE_ENCODING = 15;
+
+ /**
+ * This is only retained in off-heap subclasses. However, it's marked as
+ * Retained here so that callers are aware that the value may be retained.
+ */
+ @Override
+ @Retained
+ public Object _getValueRetain(RegionEntryContext context, boolean decompress) {
+ if (decompress) {
+ return decompress(context, _getValue());
+ } else {
+ return _getValue();
+ }
+ }
+
+ @Override
+ public void returnToPool() {
+ // noop by default
+ }
+}
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionMap.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionMap.java
new file mode 100644
index 000000000000..7661f82fdaf6
--- /dev/null
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/AbstractRegionMap.java
@@ -0,0 +1,4163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.gemstone.gemfire.internal.cache;
+
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.gemstone.gemfire.internal.cache.region.entry.RegionEntryFactoryBuilder;
+import org.apache.logging.log4j.Logger;
+
+import com.gemstone.gemfire.GemFireIOException;
+import com.gemstone.gemfire.InvalidDeltaException;
+import com.gemstone.gemfire.cache.CacheRuntimeException;
+import com.gemstone.gemfire.cache.CacheWriter;
+import com.gemstone.gemfire.cache.CacheWriterException;
+import com.gemstone.gemfire.cache.CustomEvictionAttributes;
+import com.gemstone.gemfire.cache.DiskAccessException;
+import com.gemstone.gemfire.cache.EntryExistsException;
+import com.gemstone.gemfire.cache.EntryNotFoundException;
+import com.gemstone.gemfire.cache.Operation;
+import com.gemstone.gemfire.cache.RegionDestroyedException;
+import com.gemstone.gemfire.cache.TimeoutException;
+import com.gemstone.gemfire.cache.TransactionId;
+import com.gemstone.gemfire.cache.query.IndexMaintenanceException;
+import com.gemstone.gemfire.cache.query.QueryException;
+import com.gemstone.gemfire.cache.query.internal.IndexUpdater;
+import com.gemstone.gemfire.cache.query.internal.index.IndexManager;
+import com.gemstone.gemfire.cache.query.internal.index.IndexProtocol;
+import com.gemstone.gemfire.distributed.DistributedMember;
+import com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember;
+import com.gemstone.gemfire.internal.Assert;
+import com.gemstone.gemfire.internal.ClassPathLoader;
+import com.gemstone.gemfire.internal.cache.DiskInitFile.DiskRegionFlag;
+import com.gemstone.gemfire.internal.cache.FilterRoutingInfo.FilterInfo;
+import com.gemstone.gemfire.internal.cache.delta.Delta;
+import com.gemstone.gemfire.internal.cache.ha.HAContainerWrapper;
+import com.gemstone.gemfire.internal.cache.ha.HARegionQueue;
+import com.gemstone.gemfire.internal.cache.lru.LRUEntry;
+import com.gemstone.gemfire.internal.cache.tier.sockets.CacheClientNotifier;
+import com.gemstone.gemfire.internal.cache.tier.sockets.ClientProxyMembershipID;
+import com.gemstone.gemfire.internal.cache.tier.sockets.HAEventWrapper;
+import com.gemstone.gemfire.internal.cache.versions.ConcurrentCacheModificationException;
+import com.gemstone.gemfire.internal.cache.versions.RegionVersionVector;
+import com.gemstone.gemfire.internal.cache.versions.VersionHolder;
+import com.gemstone.gemfire.internal.cache.versions.VersionSource;
+import com.gemstone.gemfire.internal.cache.versions.VersionStamp;
+import com.gemstone.gemfire.internal.cache.versions.VersionTag;
+import com.gemstone.gemfire.internal.cache.wan.GatewaySenderEventImpl;
+import com.gemstone.gemfire.internal.concurrent.MapCallbackAdapter;
+import com.gemstone.gemfire.internal.concurrent.MapResult;
+import com.gemstone.gemfire.internal.i18n.LocalizedStrings;
+import com.gemstone.gemfire.internal.logging.LogService;
+import com.gemstone.gemfire.internal.logging.log4j.LocalizedMessage;
+import com.gemstone.gemfire.internal.logging.log4j.LogMarker;
+import com.gemstone.gemfire.internal.offheap.Chunk;
+import com.gemstone.gemfire.internal.offheap.OffHeapHelper;
+import com.gemstone.gemfire.internal.offheap.OffHeapRegionEntryHelper;
+import com.gemstone.gemfire.internal.offheap.ReferenceCountHelper;
+import com.gemstone.gemfire.internal.offheap.annotations.Released;
+import com.gemstone.gemfire.internal.offheap.annotations.Retained;
+import com.gemstone.gemfire.internal.offheap.annotations.Unretained;
+import com.gemstone.gemfire.internal.sequencelog.EntryLogger;
+import com.gemstone.gemfire.internal.util.concurrent.CustomEntryConcurrentHashMap;
+import com.gemstone.gemfire.pdx.PdxInstance;
+import com.gemstone.gemfire.pdx.PdxSerializationException;
+import com.gemstone.gemfire.pdx.internal.ConvertableToBytes;
+
+/**
+ * Abstract implementation of {@link RegionMap}that has all the common
+ * behavior.
+ *
+ * @since 3.5.1
+ *
+ *
+ */
+
+//Asif: In case of sqlFabric System, we are creating a different set of RegionEntry
+// which are derived from the concrete GFE RegionEntry classes.
+// In future if any new concrete RegionEntry class is defined, the new SqlFabric
+// RegionEntry Classes need to be created. There is a junit test in sqlfabric
+// which checks for RegionEntry classes of GFE and validates the same with its
+// own classes.
+
+public abstract class AbstractRegionMap implements RegionMap {
+
+ private static final Logger logger = LogService.getLogger();
+
+ /** The underlying map for this region. */
+ protected CustomEntryConcurrentHashMap
+ *
+ * Each
+ * Note that if the value had already been written to disk, it is not
+ * written again.
+ *
+ * Caller must synchronize on entry and it is assumed the entry is evicted
+ *
+ * see #writeToDisk
+ * @throws RegionClearedException
+ */
+ public static int overflowToDisk(DiskEntry entry, LocalRegion region, EnableLRU ccHelper) throws RegionClearedException {
+ {
+ Token entryVal = entry.getValueAsToken();
+ if (entryVal == null || Token.isRemovedFromDisk(entryVal)) {
+ // Note it could be removed token now because
+ // freeAllEntriesOnDisk is not able to sync on entry
+ return 0;
+ }
+ }
+ DiskRegion dr = region.getDiskRegion();
+ final int oldSize = region.calculateRegionEntryValueSize(entry);;
+ //Asif:Get diskID . If it is null, it implies it is
+ // overflow only mode.
+ //long id = entry.getDiskId().getKeyId();
+ DiskId did = entry.getDiskId();
+ if (did == null) {
+ ((LRUEntry)entry).setDelayedDiskId(region);
+ did = entry.getDiskId();
+ }
+
+ // Notify the SQLFire IndexManager if present
+ /* final IndexUpdater indexUpdater = region.getIndexUpdater();
+ if(indexUpdater != null && dr.isSync()) {
+ indexUpdater.onOverflowToDisk(entry);
+ }*/
+
+ int change = 0;
+ boolean scheduledAsyncHere = false;
+ dr.acquireReadLock();
+ try {
+ synchronized (did) {
+ // check for a concurrent freeAllEntriesOnDisk
+ if (entry.isRemovedFromDisk()) {
+ return 0;
+ }
+
+ //TODO:Asif: Check if we need to overflow even when id is = 0
+ boolean wasAlreadyPendingAsync = did.isPendingAsync();
+ if (did.needsToBeWritten()) {
+ if (dr.isSync()) {
+ writeToDisk(entry, region, false);
+ } else if (!wasAlreadyPendingAsync) {
+ scheduledAsyncHere = true;
+ did.setPendingAsync(true);
+ } else {
+ // it may have been scheduled to be written (isBackup==true)
+ // and now we are faulting it out
+ }
+ }
+
+ boolean movedValueToDisk = false; // added for bug 41849
+
+ // If async then if it does not need to be written (because it already was)
+ // then treat it like the sync case. This fixes bug 41310
+ if (scheduledAsyncHere || wasAlreadyPendingAsync) {
+ // we call _setValue(null) after it is actually written to disk
+ change = entry.updateAsyncEntrySize(ccHelper);
+ // do the stats when it is actually written to disk
+ } else {
+ region.updateSizeOnEvict(entry.getKey(), oldSize);
+ //did.setValueSerializedSize(byteSizeOnDisk);
+ try {
+ entry.handleValueOverflow(region);
+ entry.setValueWithContext(region,null);
+ }finally {
+ entry.afterValueOverflow(region);
+ }
+ movedValueToDisk = true;
+ change = ((LRUClockNode)entry).updateEntrySize(ccHelper);
+ }
+ int valueLength = 0;
+ if (movedValueToDisk) {
+ valueLength = getValueLength(did);
+ }
+ dr.incNumEntriesInVM(-1L);
+ dr.incNumOverflowOnDisk(1L);
+ dr.incNumOverflowBytesOnDisk(valueLength);
+ incrementBucketStats(region, -1/*InVM*/, 1/*OnDisk*/, valueLength);
+ }
+ } finally {
+ dr.releaseReadLock();
+ }
+ if (scheduledAsyncHere && did.isPendingAsync()) {
+ // this needs to be done outside the above sync
+ // the version tag is null here because this method only needs
+ // to write to disk for overflow only regions, which do not need
+ // to maintain an RVV on disk.
+ scheduleAsyncWrite(new AsyncDiskEntry(region, entry, null));
+ }
+ return change;
+ }
+
+ private static void scheduleAsyncWrite(AsyncDiskEntry ade) {
+ DiskRegion dr = ade.region.getDiskRegion();
+ dr.scheduleAsyncWrite(ade);
+ }
+
+
+ public static void handleFullAsyncQueue(DiskEntry entry, LocalRegion region, VersionTag tag) {
+ DiskRegion dr = region.getDiskRegion();
+ DiskId did = entry.getDiskId();
+ synchronized (entry) {
+ dr.acquireReadLock();
+ try {
+ synchronized (did) {
+ if (did.isPendingAsync()) {
+ did.setPendingAsync(false);
+ final Token entryVal = entry.getValueAsToken();
+ final int entryValSize = region.calculateRegionEntryValueSize(entry);
+ boolean remove = false;
+ try {
+ if (Token.isRemovedFromDisk(entryVal)) {
+ // onDisk was already deced so just do the valueLength here
+ dr.incNumOverflowBytesOnDisk(-did.getValueLength());
+ incrementBucketStats(region, 0/*InVM*/, 0/*OnDisk*/,
+ -did.getValueLength());
+ dr.remove(region, entry, true, false);
+ if (dr.isBackup()) {
+ did.setKeyId(DiskRegion.INVALID_ID); // fix for bug 41340
+ }
+ remove = true;
+ } else if (Token.isInvalid(entryVal) && !dr.isBackup()) {
+ // no need to write invalid to disk if overflow only
+ } else if (entryVal != null) {
+ writeToDisk(entry, region, true);
+ } else {
+ //if we have a version tag we need to record the operation
+ //to update the RVV
+ if(tag != null) {
+ DiskEntry.Helper.doAsyncFlush(tag, region);
+ }
+ return;
+ }
+ assert !dr.isSync();
+ // Only setValue to null if this was an evict.
+ // We could just be a backup that is writing async.
+ if (!remove
+ && !Token.isInvalid(entryVal)
+ && entry instanceof LRUEntry
+ && ((LRUEntry)entry).testEvicted()) {
+ // Moved this here to fix bug 40116.
+ region.updateSizeOnEvict(entry.getKey(), entryValSize);
+ // note the old size was already accounted for
+ // onDisk was already inced so just do the valueLength here
+ dr.incNumOverflowBytesOnDisk(did.getValueLength());
+ incrementBucketStats(region, 0/*InVM*/, 0/*OnDisk*/,
+ did.getValueLength());
+ try {
+ entry.handleValueOverflow(region);
+ entry.setValueWithContext(region,null);
+ }finally {
+ entry.afterValueOverflow(region);
+ }
+ }
+
+ //See if we the entry we wrote to disk has the same tag
+ //as this entry. If not, write the tag as a conflicting operation.
+ //to update the RVV.
+ VersionStamp stamp = entry.getVersionStamp();
+ if(tag != null && stamp != null
+ && (stamp.getMemberID() != tag.getMemberID()
+ || stamp.getRegionVersion() != tag.getRegionVersion())) {
+ DiskEntry.Helper.doAsyncFlush(tag, region);
+ }
+ } catch (RegionClearedException ignore) {
+ // no need to do the op since it was clobbered by a region clear
+ }
+ } else {
+ //if we have a version tag we need to record the operation
+ //to update the RVV, even if we don't write the entry
+ if(tag != null) {
+ DiskEntry.Helper.doAsyncFlush(tag, region);
+ }
+ }
+ }
+ } finally {
+ dr.releaseReadLock();
+ }
+ } // sync entry
+ }
+
+ public static void doAsyncFlush(VersionTag tag, LocalRegion region) {
+ if (region.isThisRegionBeingClosedOrDestroyed()) return;
+ DiskRegion dr = region.getDiskRegion();
+ if (!dr.isBackup()) {
+ return;
+ }
+ assert !dr.isSync();
+ dr.acquireReadLock();
+ try {
+ dr.getDiskStore().putVersionTagOnly(region, tag, true);
+ } finally {
+ dr.releaseReadLock();
+ }
+ }
+
+ /**
+ * Flush an entry that was previously scheduled to be written to disk.
+ * @param tag
+ * @since prPersistSprint1
+ */
+ public static void doAsyncFlush(DiskEntry entry, LocalRegion region, VersionTag tag) {
+ if (region.isThisRegionBeingClosedOrDestroyed()) return;
+ DiskRegion dr = region.getDiskRegion();
+ dr.setClearCountReference();
+ synchronized (entry) { // fixes 40116
+ // If I don't sync the entry and this method ends up doing an eviction
+ // thus setting value to null
+ // some other thread is free to fetch the value while the entry is synced
+ // and think it has removed it or replaced it. This results in updateSizeOn*
+ // being called twice for the same value (once when it is evicted and once
+ // when it is removed/updated).
+ try {
+ dr.acquireReadLock();
+ try {
+ DiskId did = entry.getDiskId();
+ synchronized (did) {
+ if (did.isPendingAsync()) {
+ did.setPendingAsync(false);
+ final Token entryVal = entry.getValueAsToken();
+ final int entryValSize = region.calculateRegionEntryValueSize(entry);
+ boolean remove = false;
+ try {
+ if (Token.isRemovedFromDisk(entryVal)) {
+ if (region.isThisRegionBeingClosedOrDestroyed()) return;
+ // onDisk was already deced so just do the valueLength here
+ dr.incNumOverflowBytesOnDisk(-did.getValueLength());
+ incrementBucketStats(region, 0/*InVM*/, 0/*OnDisk*/,
+ -did.getValueLength());
+ dr.remove(region, entry, true, false);
+ if (dr.isBackup()) {
+ did.setKeyId(DiskRegion.INVALID_ID); // fix for bug 41340
+ }
+ remove = true;
+ } else if ((Token.isInvalid(entryVal) || entryVal == Token.TOMBSTONE) && !dr.isBackup()) {
+ // no need to write invalid or tombstones to disk if overflow only
+ } else if (entryVal != null) {
+ writeToDisk(entry, region, true);
+ } else {
+ // @todo why would we have a null value here?
+ // I'm seeing it show up in tests:
+// java.lang.IllegalArgumentException: Must not serialize null in this context.
+// at com.gemstone.gemfire.internal.cache.EntryEventImpl.serialize(EntryEventImpl.java:1024)
+// at com.gemstone.gemfire.internal.cache.DiskEntry$Helper.writeToDisk(DiskEntry.java:351)
+// at com.gemstone.gemfire.internal.cache.DiskEntry$Helper.doAsyncFlush(DiskEntry.java:683)
+// at com.gemstone.gemfire.internal.cache.DiskRegion$FlusherThread.run(DiskRegion.java:1055)
+ //if we have a version tag we need to record the operation
+ //to update the RVV
+ if(tag != null) {
+ DiskEntry.Helper.doAsyncFlush(tag, region);
+ }
+ return;
+ }
+ assert !dr.isSync();
+ // Only setValue to null if this was an evict.
+ // We could just be a backup that is writing async.
+ if (!remove
+ && !Token.isInvalid(entryVal)
+ && (entryVal != Token.TOMBSTONE)
+ && entry instanceof LRUEntry
+ && ((LRUEntry)entry).testEvicted()) {
+ // Moved this here to fix bug 40116.
+ region.updateSizeOnEvict(entry.getKey(), entryValSize);
+ // note the old size was already accounted for
+ // onDisk was already inced so just do the valueLength here
+ dr.incNumOverflowBytesOnDisk(did.getValueLength());
+ incrementBucketStats(region, 0/*InVM*/, 0/*OnDisk*/,
+ did.getValueLength());
+ try {
+ entry.handleValueOverflow(region);
+ entry.setValueWithContext(region,null);
+ }finally {
+ entry.afterValueOverflow(region);
+ }
+ }
+ } catch (RegionClearedException ignore) {
+ // no need to do the op since it was clobbered by a region clear
+ }
+
+ //See if we the entry we wrote to disk has the same tag
+ //as this entry. If not, write the tag as a conflicting operation.
+ //to update the RVV.
+ VersionStamp stamp = entry.getVersionStamp();
+ if(tag != null && stamp != null
+ && (stamp.getMemberID() != tag.getMemberID()
+ || stamp.getRegionVersion() != tag.getRegionVersion())) {
+ DiskEntry.Helper.doAsyncFlush(tag, region);
+ }
+ } else {
+ //if we have a version tag we need to record the operation
+ //to update the RVV
+ if(tag != null) {
+ DiskEntry.Helper.doAsyncFlush(tag, region);
+ }
+ }
+ }
+ } finally {
+ dr.releaseReadLock();
+ }
+ } finally {
+ dr.removeClearCountReference();
+ }
+ } // sync entry
+ }
+
+ /**
+ * Removes the key/value pair in the given entry from disk
+ *
+ * @throws RegionClearedException If the operation is aborted due to a clear
+ * @see DiskRegion#remove
+ */
+ public static void removeFromDisk(DiskEntry entry, LocalRegion region, boolean isClear) throws RegionClearedException {
+ removeFromDisk(entry, region, true, false, isClear);
+ }
+ private static void removeFromDisk(DiskEntry entry, LocalRegion region,
+ boolean checkValue, boolean valueWasNull, boolean isClear) throws RegionClearedException {
+ DiskRegion dr = region.getDiskRegion();
+
+ //If we have concurrency checks enabled for a persistent region, we need
+ //to add an entry to the async queue for every update to maintain the RVV
+ boolean maintainRVV = region.concurrencyChecksEnabled && dr.isBackup();
+
+ DiskId did = entry.getDiskId();
+ VersionTag tag = null;
+ Object syncObj = did;
+ if (did == null) {
+ syncObj = entry;
+ }
+ boolean scheduledAsyncHere = false;
+ if (syncObj == did) {
+ dr.acquireReadLock();
+ }
+ try {
+ synchronized (syncObj) {
+
+ if (did == null || (dr.isBackup() && did.getKeyId()== DiskRegion.INVALID_ID)) {
+ // Not on disk yet
+ dr.incNumEntriesInVM(-1L);
+ incrementBucketStats(region, -1/*InVM*/, 0/*OnDisk*/, 0);
+ dr.unscheduleAsyncWrite(did);
+ return;
+ }
+ //Asif: This will convert the -ve OplogKeyId to positive as part of fixing
+ //Bug # 39989
+ did.unmarkForWriting();
+
+ //System.out.println("DEBUG: removeFromDisk doing remove(" + id + ")");
+ int oldValueLength = 0;
+ if (dr.isSync() || isClear) {
+ oldValueLength = did.getValueLength();
+ dr.remove(region, entry, false, isClear);
+ if (dr.isBackup()) {
+ did.setKeyId(DiskRegion.INVALID_ID); // fix for bug 41340
+ }
+ //If this is a clear, we should unschedule the async write for this
+ //entry
+ did.setPendingAsync(false);
+ } else {
+ if (!did.isPendingAsync() || maintainRVV) {
+ scheduledAsyncHere = true;
+ did.setPendingAsync(true);
+ VersionStamp stamp = entry.getVersionStamp();
+ if(stamp != null) {
+ tag = stamp.asVersionTag();
+ }
+ }
+ }
+ if (checkValue) {
+ valueWasNull = entry.isValueNull();
+ entry._removePhase1();
+ }
+ if (valueWasNull) {
+ dr.incNumOverflowOnDisk(-1L);
+ dr.incNumOverflowBytesOnDisk(-oldValueLength);
+ incrementBucketStats(region, 0/*InVM*/, -1/*OnDisk*/, -oldValueLength);
+ }
+ else {
+ dr.incNumEntriesInVM(-1L);
+ incrementBucketStats(region, -1/*InVM*/, 0/*OnDisk*/, 0);
+ if (!dr.isSync()) {
+ // we are going to do an async remove of an entry that is not currently
+ // overflowed to disk so we don't want to count its value length as being
+ // on disk when we finally do the async op. So we clear it here.
+ did.setValueLength(0);
+ }
+ }
+ }
+ } finally {
+ if (syncObj == did) {
+ dr.releaseReadLock();
+ }
+ }
+ if (scheduledAsyncHere && did.isPendingAsync()) {
+ // do this outside the sync
+ scheduleAsyncWrite(new AsyncDiskEntry(region, entry, tag));
+ }
+ }
+
+ /**
+ * @param entry
+ * @param region
+ * @param tag
+ */
+ public static void updateVersionOnly(DiskEntry entry, LocalRegion region,
+ VersionTag tag) {
+ DiskRegion dr = region.getDiskRegion();
+ if (!dr.isBackup()) {
+ return;
+ }
+
+ assert tag != null && tag.getMemberID()!=null;
+ boolean scheduleAsync = false;
+ DiskId did = entry.getDiskId();
+ Object syncObj = did;
+ if (syncObj == null) {
+ syncObj = entry;
+ }
+ if (syncObj == did) {
+ dr.acquireReadLock();
+ }
+ try {
+ synchronized (syncObj) {
+ if (dr.isSync()) {
+ dr.getDiskStore().putVersionTagOnly(region, tag, false);
+ } else {
+ scheduleAsync = true;
+ }
+ }
+ } finally {
+ if (syncObj == did) {
+ dr.releaseReadLock();
+ }
+ }
+ if (scheduleAsync) {
+ // this needs to be done outside the above sync
+ scheduleAsyncWrite(new AsyncDiskEntry(region, tag));
+ }
+ }
+
+ }
+
+ /**
+ * A marker object for an entry that has been recovered from disk.
+ * It is handled specially when it is placed in a region.
+ */
+ public static class RecoveredEntry {
+
+ /** The disk id of the entry being recovered */
+ private final long recoveredKeyId;
+
+ /** The value of the recovered entry */
+ private final Object value;
+
+ private final long offsetInOplog;
+ private final byte userBits;
+ private final int valueLength;
+
+ private long oplogId;
+ private VersionTag tag;
+
+ /**
+ * Only for this constructor, the value is not loaded into the region & it is lying
+ * on the oplogs. Since Oplogs rely on DiskId to furnish user bits so as to correctly
+ * interpret bytes, the userbit needs to be set correctly here.
+ */
+ public RecoveredEntry(long keyId, long oplogId, long offsetInOplog,
+ byte userBits, int valueLength) {
+ this(-keyId, oplogId, offsetInOplog, userBits, valueLength, null);
+ }
+
+ public RecoveredEntry(long keyId, long oplogId, long offsetInOplog,
+ byte userBits, int valueLength, Object value) {
+ this.recoveredKeyId = keyId;
+ this.value = value;
+ this.oplogId = oplogId;
+ this.offsetInOplog = offsetInOplog;
+ this.userBits = EntryBits.setRecoveredFromDisk(userBits, true);
+ this.valueLength = valueLength;
+ }
+
+ /**
+ * Returns the disk id of the entry being recovered
+ */
+ public long getRecoveredKeyId() {
+ return this.recoveredKeyId;
+ }
+ /**
+ * Returns the value of the recovered entry. Note that if the
+ * disk id is < 0 then the value has not been faulted in and
+ * this method will return null.
+ */
+ public Object getValue() {
+ return this.value;
+ }
+ /**
+ *
+ * @return byte indicating the user bits. The correct value is returned only in the specific case of
+ * entry recovered from oplog ( & not rolled to Htree) & the RECOVER_VALUES flag is false . In other cases
+ * the exact value is not needed
+ */
+ public byte getUserBits() {
+ return this.userBits;
+ }
+ public int getValueLength() {
+ return this.valueLength;
+ }
+ public long getOffsetInOplog() {
+ return offsetInOplog;
+ }
+ public long getOplogId() {
+ return this.oplogId;
+ }
+
+ public void setOplogId(long v) {
+ this.oplogId = v;
+ }
+ public VersionTag getVersionTag() {
+ return this.tag;
+ }
+ public void setVersionTag(VersionTag tag) {
+ this.tag = tag;
+ }
+ }
+}
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DistributedRegion.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DistributedRegion.java
new file mode 100644
index 000000000000..fcfc2f333cf2
--- /dev/null
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/DistributedRegion.java
@@ -0,0 +1,4309 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.gemstone.gemfire.internal.cache;
+
+import static com.gemstone.gemfire.internal.offheap.annotations.OffHeapIdentifier.ABSTRACT_REGION_ENTRY_FILL_IN_VALUE;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArraySet;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+
+import org.apache.logging.log4j.Logger;
+
+import com.gemstone.gemfire.CancelException;
+import com.gemstone.gemfire.InternalGemFireError;
+import com.gemstone.gemfire.InvalidDeltaException;
+import com.gemstone.gemfire.SystemFailure;
+import com.gemstone.gemfire.cache.CacheClosedException;
+import com.gemstone.gemfire.cache.CacheListener;
+import com.gemstone.gemfire.cache.CacheLoader;
+import com.gemstone.gemfire.cache.CacheLoaderException;
+import com.gemstone.gemfire.cache.CacheWriter;
+import com.gemstone.gemfire.cache.CacheWriterException;
+import com.gemstone.gemfire.cache.DataPolicy;
+import com.gemstone.gemfire.cache.DiskAccessException;
+import com.gemstone.gemfire.cache.EntryNotFoundException;
+import com.gemstone.gemfire.cache.LossAction;
+import com.gemstone.gemfire.cache.MembershipAttributes;
+import com.gemstone.gemfire.cache.Operation;
+import com.gemstone.gemfire.cache.RegionAccessException;
+import com.gemstone.gemfire.cache.RegionAttributes;
+import com.gemstone.gemfire.cache.RegionDestroyedException;
+import com.gemstone.gemfire.cache.RegionDistributionException;
+import com.gemstone.gemfire.cache.RegionMembershipListener;
+import com.gemstone.gemfire.cache.ResumptionAction;
+import com.gemstone.gemfire.cache.RoleException;
+import com.gemstone.gemfire.cache.TimeoutException;
+import com.gemstone.gemfire.cache.TransactionId;
+import com.gemstone.gemfire.cache.asyncqueue.internal.AsyncEventQueueImpl;
+import com.gemstone.gemfire.cache.execute.Function;
+import com.gemstone.gemfire.cache.execute.FunctionException;
+import com.gemstone.gemfire.cache.execute.ResultCollector;
+import com.gemstone.gemfire.cache.execute.ResultSender;
+import com.gemstone.gemfire.cache.persistence.PersistentReplicatesOfflineException;
+import com.gemstone.gemfire.cache.query.internal.IndexUpdater;
+import com.gemstone.gemfire.cache.wan.GatewaySender;
+import com.gemstone.gemfire.distributed.DistributedLockService;
+import com.gemstone.gemfire.distributed.DistributedMember;
+import com.gemstone.gemfire.distributed.LockServiceDestroyedException;
+import com.gemstone.gemfire.distributed.Role;
+import com.gemstone.gemfire.distributed.internal.DM;
+import com.gemstone.gemfire.distributed.internal.DistributionAdvisee;
+import com.gemstone.gemfire.distributed.internal.DistributionAdvisor;
+import com.gemstone.gemfire.distributed.internal.DistributionAdvisor.Profile;
+import com.gemstone.gemfire.distributed.internal.DistributionAdvisor.ProfileVisitor;
+import com.gemstone.gemfire.distributed.internal.DistributionConfig;
+import com.gemstone.gemfire.distributed.internal.MembershipListener;
+import com.gemstone.gemfire.distributed.internal.ReplyProcessor21;
+import com.gemstone.gemfire.distributed.internal.locks.DLockRemoteToken;
+import com.gemstone.gemfire.distributed.internal.locks.DLockService;
+import com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember;
+import com.gemstone.gemfire.internal.Assert;
+import com.gemstone.gemfire.internal.cache.CacheDistributionAdvisor.CacheProfile;
+import com.gemstone.gemfire.internal.cache.InitialImageOperation.GIIStatus;
+import com.gemstone.gemfire.internal.cache.RemoteFetchVersionMessage.FetchVersionResponse;
+import com.gemstone.gemfire.internal.cache.control.InternalResourceManager.ResourceType;
+import com.gemstone.gemfire.internal.cache.control.MemoryEvent;
+import com.gemstone.gemfire.internal.cache.execute.DistributedRegionFunctionExecutor;
+import com.gemstone.gemfire.internal.cache.execute.DistributedRegionFunctionResultSender;
+import com.gemstone.gemfire.internal.cache.execute.DistributedRegionFunctionResultWaiter;
+import com.gemstone.gemfire.internal.cache.execute.FunctionStats;
+import com.gemstone.gemfire.internal.cache.execute.LocalResultCollector;
+import com.gemstone.gemfire.internal.cache.execute.RegionFunctionContextImpl;
+import com.gemstone.gemfire.internal.cache.execute.ServerToClientFunctionResultSender;
+import com.gemstone.gemfire.internal.cache.lru.LRUEntry;
+import com.gemstone.gemfire.internal.cache.persistence.CreatePersistentRegionProcessor;
+import com.gemstone.gemfire.internal.cache.persistence.PersistenceAdvisor;
+import com.gemstone.gemfire.internal.cache.persistence.PersistenceAdvisorImpl;
+import com.gemstone.gemfire.internal.cache.persistence.PersistentMemberID;
+import com.gemstone.gemfire.internal.cache.persistence.PersistentMemberManager;
+import com.gemstone.gemfire.internal.cache.persistence.PersistentMemberView;
+import com.gemstone.gemfire.internal.cache.tier.sockets.ClientProxyMembershipID;
+import com.gemstone.gemfire.internal.cache.tier.sockets.VersionedObjectList;
+import com.gemstone.gemfire.internal.cache.versions.ConcurrentCacheModificationException;
+import com.gemstone.gemfire.internal.cache.versions.RegionVersionVector;
+import com.gemstone.gemfire.internal.cache.versions.VersionSource;
+import com.gemstone.gemfire.internal.cache.versions.VersionTag;
+import com.gemstone.gemfire.internal.cache.wan.AbstractGatewaySender;
+import com.gemstone.gemfire.internal.cache.wan.AbstractGatewaySenderEventProcessor;
+import com.gemstone.gemfire.internal.cache.wan.AsyncEventQueueConfigurationException;
+import com.gemstone.gemfire.internal.cache.wan.GatewaySenderConfigurationException;
+import com.gemstone.gemfire.internal.cache.wan.parallel.ConcurrentParallelGatewaySenderQueue;
+import com.gemstone.gemfire.internal.i18n.LocalizedStrings;
+import com.gemstone.gemfire.internal.logging.LogService;
+import com.gemstone.gemfire.internal.logging.log4j.LocalizedMessage;
+import com.gemstone.gemfire.internal.offheap.Chunk;
+import com.gemstone.gemfire.internal.offheap.OffHeapHelper;
+import com.gemstone.gemfire.internal.offheap.annotations.Released;
+import com.gemstone.gemfire.internal.offheap.annotations.Retained;
+import com.gemstone.gemfire.internal.sequencelog.RegionLogger;
+import com.gemstone.gemfire.internal.util.concurrent.StoppableCountDownLatch;
+import com.gemstone.gemfire.i18n.StringId;
+/**
+ *
+ */
+@SuppressWarnings("deprecation")
+public class DistributedRegion extends LocalRegion implements
+ CacheDistributionAdvisee
+{
+ private static final Logger logger = LogService.getLogger();
+
+ /** causes cache profile to be added to afterRemoteRegionCreate notification for testing */
+ public static boolean TEST_HOOK_ADD_PROFILE = false;
+
+ /** Used to sync accesses to this.dlockService to allow lazy construction */
+ private final Object dlockMonitor = new Object();
+
+ final CacheDistributionAdvisor distAdvisor;
+
+ /**
+ * @guarded.By {@link #dlockMonitor}
+ */
+ private DistributedLockService dlockService;
+
+ protected final AdvisorListener advisorListener = new AdvisorListener();
+
+ /** Set of currently missing required roles */
+ protected final HashSet missingRequiredRoles = new HashSet();
+
+ /** True if this region is currently missing any required roles */
+ protected volatile boolean isMissingRequiredRoles = false;
+
+ /**
+ * True if this region is has any required roles defined and the LossAction is
+ * either NO_ACCESS or LIMITED_ACCESS. Reliability checks will only happen if
+ * this is true.
+ */
+ private final boolean requiresReliabilityCheck;
+
+ /**
+ * Provides a queue for reliable message delivery
+ *
+ * @since 5.0
+ */
+ protected final ReliableMessageQueue rmq;
+
+ /**
+ * Latch that is opened after initialization waits for required roles up to
+ * the member-timeout .
+ */
+ protected final StoppableCountDownLatch initializationLatchAfterMemberTimeout;
+
+ private final PersistenceAdvisor persistenceAdvisor;
+
+ private final PersistentMemberID persistentId;
+
+ /**
+ * This boolean is set to false when this region
+ * is non-persistent, but there are persistent members in the distributed system
+ * to which all region modifications should be forwarded
+ * see bug 45186
+ */
+ private volatile boolean generateVersionTag = true;
+
+ /** Tests can set this to true and ignore reliability triggered reconnects */
+ public static boolean ignoreReconnect = false;
+
+ /**
+ * Lock to prevent multiple threads on this member from performing
+ * a clear at the same time.
+ */
+ private final Object clearLock = new Object();
+
+ private static AtomicBoolean loggedNetworkPartitionWarning = new AtomicBoolean(false);
+
+ /** Creates a new instance of DistributedRegion */
+ protected DistributedRegion(String regionName, RegionAttributes attrs,
+ LocalRegion parentRegion, GemFireCacheImpl cache,
+ InternalRegionArguments internalRegionArgs) {
+ super(regionName, attrs, parentRegion, cache, internalRegionArgs);
+ this.initializationLatchAfterMemberTimeout = new StoppableCountDownLatch(
+ getCancelCriterion(), 1);
+ this.distAdvisor = createDistributionAdvisor(internalRegionArgs);
+
+ if (getDistributionManager().getConfig().getEnableNetworkPartitionDetection()
+ && !isInternalRegion() && !attrs.getScope().isAck() && !doesNotDistribute() && attrs.getDataPolicy().withStorage()) {
+ logger.warn(LocalizedMessage.create(LocalizedStrings.DistributedRegion_REGION_0_1_SPLITBRAIN_CONFIG_WARNING,
+ new Object[] { regionName, attrs.getScope() }));
+ }
+ if (!getDistributionManager().getConfig().getEnableNetworkPartitionDetection()
+ && attrs.getDataPolicy().withPersistence() && !loggedNetworkPartitionWarning.getAndSet(true)) {
+ logger.warn(LocalizedMessage.create(
+ LocalizedStrings.DistributedRegion_REGION_0_ENABLE_NETWORK_PARTITION_WARNING,
+ new Object[] { regionName, attrs.getScope() }));
+ }
+
+ boolean setRequiresReliabilityCheck = attrs.getMembershipAttributes()
+ .hasRequiredRoles()
+ &&
+ // note that the following includes NO_ACCESS, LIMITED_ACCESS,
+ !attrs.getMembershipAttributes().getLossAction().isAllAccess()
+ && !attrs.getMembershipAttributes().getLossAction().isReconnect();
+
+ // this optimization is safe for as long as Roles and Required Roles are
+ // immutable
+ // if this VM fulfills all required roles, make requiresReliabilityCheck
+ // false
+ Set reqRoles = new HashSet(attrs.getMembershipAttributes()
+ .getRequiredRoles());
+ reqRoles.removeAll(getSystem().getDistributedMember().getRoles());
+ if (reqRoles.isEmpty()) {
+ setRequiresReliabilityCheck = false;
+ }
+
+ this.requiresReliabilityCheck = setRequiresReliabilityCheck;
+
+ {
+ ReliableMessageQueue tmp = null;
+ if (this.requiresReliabilityCheck) {
+ // if
+ // (attrs.getMembershipAttributes().getLossAction().isAllAccessWithQueuing())
+ // {
+ // tmp = cache.getReliableMessageQueueFactory().create(this);
+ // }
+ }
+ this.rmq = tmp;
+ }
+
+ if(internalRegionArgs.isUsedForPartitionedRegionBucket()) {
+ this.persistenceAdvisor = internalRegionArgs.getPersistenceAdvisor();
+ } else if (this.allowsPersistence()){
+ //TODO prpersist - using this lock service is a hack. Maybe? Or maybe
+ //it's ok if we have one (rarely used) lock service for many operations?
+ //What does the resource manager do?
+ DistributedLockService dl = cache.getPartitionedRegionLockService();
+ try {
+ //TODO prpersist - this is just a quick and dirty storage mechanism so that
+ //I can test the storage.
+ DiskRegionStats diskStats;
+ PersistentMemberView storage;
+ if(getDataPolicy().withPersistence()) {
+ storage = getDiskRegion();
+ diskStats = getDiskRegion().getStats();
+ } else {
+ storage = new InMemoryPersistentMemberView();
+ diskStats = null;
+ }
+ PersistentMemberManager memberManager = cache.getPersistentMemberManager();
+ this.persistenceAdvisor = new PersistenceAdvisorImpl(distAdvisor, dl, storage, this.getFullPath(), diskStats, memberManager);
+ } catch (Exception e) {
+ throw new InternalGemFireError("Couldn't recover persistence");
+ }
+ } else {
+ this.persistenceAdvisor = null;
+ }
+ if(this.persistenceAdvisor != null) {
+ this.persistentId = persistenceAdvisor.generatePersistentID();
+ } else {
+ this.persistentId = null;
+ }
+
+ }
+
+ @Override
+ public void createEventTracker() {
+ this.eventTracker = new EventTracker(this);
+ this.eventTracker.start();
+ }
+
+ /**
+ * Intended for used during construction of a DistributedRegion
+ *
+ * @return the advisor to be used by the region
+ */
+ protected CacheDistributionAdvisor createDistributionAdvisor(InternalRegionArguments internalRegionArgs) {
+ return CacheDistributionAdvisor.createCacheDistributionAdvisor(this); // Warning: potential early escape of object before full construction
+ }
+
+ /**
+ * Does this region support persistence?
+ */
+ public boolean allowsPersistence() {
+ return true;
+ }
+
+ @Override
+ public boolean requiresOneHopForMissingEntry(EntryEventImpl event) {
+ // received from another member - don't use one-hop
+ if (event.isOriginRemote()) {
+ return false;
+ }
+ // local ops aren't distributed
+ if (event.getOperation().isLocal()) {
+ return false;
+ }
+ // if it already has a valid version tag it can go out with a DistributedCacheOperation
+ if (event.getVersionTag() != null && event.getVersionTag().getRegionVersion() > 0) {
+ return false;
+ }
+ // if we're not allowed to generate a version tag we need to send it to someone who can
+ if (!this.generateVersionTag) {
+ return true;
+ }
+ return this.concurrencyChecksEnabled &&
+ (this.srp == null) &&
+ !isTX() &&
+ this.scope.isDistributed() &&
+ !this.dataPolicy.withReplication();
+ }
+
+
+ /**
+ * @see LocalRegion#virtualPut(EntryEventImpl, boolean, boolean, Object,
+ * boolean, long, boolean)
+ */
+ @Override
+ protected
+ boolean virtualPut(EntryEventImpl event,
+ boolean ifNew,
+ boolean ifOld,
+ Object expectedOldValue,
+ boolean requireOldValue,
+ long lastModified,
+ boolean overwriteDestroyed)
+ throws TimeoutException,
+ CacheWriterException {
+ final boolean isTraceEnabled = logger.isTraceEnabled();
+
+ Lock dlock = null;
+ if (this.scope.isGlobal() && // lock only applies to global scope
+ !event.isOriginRemote() && // only if operation originating locally
+ !event.isNetSearch() && // search and load processor handles own locking
+ !event.isNetLoad() &&
+ // @todo darrel/kirk: what about putAll?
+ !event.isLocalLoad() &&
+ !event.isSingleHopPutOp()) { // Single Hop Op means dlock is already taken at origin node.
+ dlock = this.getDistributedLockIfGlobal(event.getKey());
+ }
+ if (isTraceEnabled) {
+ logger.trace("virtualPut invoked for event {}", event);
+ }
+ try {
+ if (!hasSeenEvent(event)) {
+ if (this.requiresOneHopForMissingEntry(event)) {
+ // bug #45704: see if a one-hop must be done for this operation
+ RegionEntry re = getRegionEntry(event.getKey());
+ if (re == null /*|| re.isTombstone()*/ || !this.generateVersionTag) {
+ if (!event.isBulkOpInProgress() || this.dataPolicy.withStorage()) {
+ // putAll will send a single one-hop for empty regions. for other missing entries
+ // we need to get a valid version number before modifying the local cache
+ boolean didDistribute = RemotePutMessage.distribute(event, lastModified,
+ false, false, expectedOldValue, requireOldValue, !this.generateVersionTag);
+
+ if (!didDistribute && isTraceEnabled) {
+ logger.trace("Unable to perform one-hop messaging");
+ }
+ if (!this.generateVersionTag && !didDistribute) {
+ throw new PersistentReplicatesOfflineException();
+ }
+ if (didDistribute) {
+ if (isTraceEnabled) {
+ logger.trace("Event after remotePut operation: {}", event);
+ }
+ if (event.getVersionTag() == null) {
+ // if the event wasn't applied by the one-hop replicate it will not have a version tag
+ // and so should not be applied to this cache
+ return false;
+ }
+ }
+ }
+ }
+ }
+ return super.virtualPut(event,
+ ifNew,
+ ifOld,
+ expectedOldValue,
+ requireOldValue,
+ lastModified,
+ overwriteDestroyed);
+ }
+ else {
+ if (event.getDeltaBytes() != null && event.getRawNewValue() == null) {
+ // This means that this event has delta bytes but no full value.
+ // Request the full value of this event.
+ // The value in this vm may not be same as this event's value.
+ throw new InvalidDeltaException(
+ "Cache encountered replay of event containing delta bytes for key "
+ + event.getKey());
+ }
+ // if the listeners have already seen this event, then it has already
+ // been successfully applied to the cache. Distributed messages and
+ // return
+ if (isTraceEnabled) {
+ logger.trace("DR.virtualPut: this cache has already seen this event {}", event);
+ }
+
+ // Gester, Fix 39014: when hasSeenEvent, put will still distribute
+ // event, but putAll did not. We add the logic back here, not to put
+ // back into DR.distributeUpdate() because we moved this part up into
+ // LR.basicPutPart3 in purpose. Reviewed by Bruce.
+ if (event.isBulkOpInProgress() && !event.isOriginRemote()) {
+ event.getPutAllOperation().addEntry(event, true);
+ }
+
+ /* doing this so that other VMs will apply this no matter what. If it
+ * is an "update" they will not apply it if they don't have the key.
+ * Because this is probably a retry, it will never get applied to this
+ * local AbstractRegionMap, and so will never be flipped to a 'create'
+ */
+ event.makeCreate();
+ if (!getConcurrencyChecksEnabled() || event.hasValidVersionTag()) {
+ distributeUpdate(event, lastModified, ifNew, ifOld, expectedOldValue, requireOldValue);
+ event.invokeCallbacks(this,true, true);
+ }
+ return true;
+ }
+ }
+ finally {
+ if (dlock != null) {
+ dlock.unlock();
+ }
+ }
+ }
+
+ @Override
+ protected RegionEntry basicPutEntry(EntryEventImpl event, long lastModified)
+ throws TimeoutException, CacheWriterException {
+
+ final boolean isTraceEnabled = logger.isTraceEnabled();
+
+ if (isTraceEnabled) {
+ logger.trace("basicPutEntry invoked for event {}", event);
+ }
+ if (this.requiresOneHopForMissingEntry(event)) {
+ // bug #45704: see if a one-hop must be done for this operation
+ RegionEntry re = getRegionEntry(event.getKey());
+ if (re == null /*|| re.isTombstone()*/ || !this.generateVersionTag) {
+ final boolean ifNew = false;
+ final boolean ifOld = false;
+ boolean didDistribute = RemotePutMessage.distribute(event, lastModified,
+ ifNew, ifOld, null, false, !this.generateVersionTag);
+ if (!this.generateVersionTag && !didDistribute) {
+ throw new PersistentReplicatesOfflineException();
+ }
+ if (didDistribute && isTraceEnabled) {
+ logger.trace("Event after remotePut for basicPutEntry: {}", event);
+ }
+ }
+ }
+ return super.basicPutEntry(event, lastModified);
+ }
+
+ @Override
+ public void performPutAllEntry(EntryEventImpl event) {
+ /*
+ * force shared data view so that we just do the virtual op, accruing things in the put all operation for later
+ */
+ if(isTX()) {
+ event.getPutAllOperation().addEntry(event);
+ } else {
+ getSharedDataView().putEntry(event, false, false, null, false, 0L, false);
+ }
+ }
+
+ @Override
+ public void performRemoveAllEntry(EntryEventImpl event) {
+ // force shared data view so that we just do the virtual op, accruing things in the bulk operation for later
+ if(isTX()) {
+ event.getRemoveAllOperation().addEntry(event);
+ } else {
+ basicDestroy(event, true, null);
+ //getSharedDataView().destroyExistingEntry(event, true, null);
+ }
+ }
+
+ /**
+ * distribution and listener notification
+ */
+ @Override
+ public void basicPutPart3(EntryEventImpl event, RegionEntry entry,
+ boolean isInitialized, long lastModified, boolean invokeCallbacks,
+ boolean ifNew, boolean ifOld, Object expectedOldValue,
+ boolean requireOldValue) {
+
+ distributeUpdate(event, lastModified, false, false, null, false);
+ super.basicPutPart3(event, entry, isInitialized, lastModified,
+ invokeCallbacks, ifNew, ifOld, expectedOldValue, requireOldValue);
+ }
+
+ /** distribute an update operation */
+ protected void distributeUpdate(EntryEventImpl event, long lastModified, boolean ifNew, boolean ifOld, Object expectedOldValue, boolean requireOldValue) {
+ // an update from a netSearch is not distributed
+ if (!event.isOriginRemote() && !event.isNetSearch() && !event.isBulkOpInProgress()) {
+ boolean distribute = true;
+ if (event.getInhibitDistribution()) {
+ // this has already been distributed by a one-hop operation
+ distribute = false;
+ }
+ if (distribute) {
+ UpdateOperation op = new UpdateOperation(event, lastModified);
+ if (logger.isTraceEnabled()) {
+ logger.trace("distributing operation for event : {} : for region : {}", event, this.getName());
+ }
+ op.distribute();
+ }
+ }
+ }
+
+ protected void setGeneratedVersionTag(boolean generateVersionTag) {
+ // there is at-least one other persistent member, so turn on concurrencyChecks
+ enableConcurrencyChecks();
+
+ this.generateVersionTag = generateVersionTag;
+ }
+
+ protected boolean getGenerateVersionTag() {
+ return this.generateVersionTag;
+ }
+
+ @Override
+ protected boolean shouldGenerateVersionTag(RegionEntry entry, EntryEventImpl event) {
+ if (logger.isTraceEnabled()) {
+ logger.trace("shouldGenerateVersionTag this.generateVersionTag={} ccenabled={} dataPolicy={} event:{}",
+ this.generateVersionTag, this.concurrencyChecksEnabled, this.dataPolicy, event);
+ }
+ if (!this.concurrencyChecksEnabled || this.dataPolicy == DataPolicy.EMPTY || !this.generateVersionTag) {
+ return false;
+ }
+ if (this.srp != null) { // client
+ return false;
+ }
+ if (event.getVersionTag() != null && !event.getVersionTag().isGatewayTag()) {
+ return false;
+ }
+ if (event.getOperation().isLocal()) { // bug #45402 - localDestroy generated a version tag
+ return false;
+ }
+ if (!event.isOriginRemote() && this.dataPolicy.withReplication()) {
+ return true;
+ }
+ if (!this.dataPolicy.withReplication() && !this.dataPolicy.withPersistence()) {
+ if (!entry.getVersionStamp().hasValidVersion()) {
+ // do not generate a version stamp in a region that has no replication if it's not based
+ // on an existing version from a replicate region
+ return false;
+ }
+ return true;
+ }
+ if (!event.isOriginRemote() && event.getDistributedMember() != null) {
+ if (!event.getDistributedMember().equals(this.getMyId())) {
+ return event.getVersionTag() == null; // one-hop remote message
+ }
+ }
+ return false;
+ }
+ /**
+ * Throws RegionAccessException if required roles are missing and the
+ * LossAction is NO_ACCESS
+ *
+ * @throws RegionAccessException
+ * if required roles are missing and the LossAction is NO_ACCESS
+ */
+ @Override
+ protected void checkForNoAccess()
+ {
+ if (this.requiresReliabilityCheck && this.isMissingRequiredRoles) {
+ if (getMembershipAttributes().getLossAction().isNoAccess()) {
+ synchronized (this.missingRequiredRoles) {
+ if (!this.isMissingRequiredRoles)
+ return;
+ Set roles = Collections.unmodifiableSet(new HashSet(
+ this.missingRequiredRoles));
+ throw new RegionAccessException(LocalizedStrings.DistributedRegion_OPERATION_IS_DISALLOWED_BY_LOSSACTION_0_BECAUSE_THESE_REQUIRED_ROLES_ARE_MISSING_1.toLocalizedString(new Object[] {getMembershipAttributes().getLossAction(), roles}), getFullPath(), roles);
+ }
+ }
+ }
+ }
+
+ /**
+ * Throws RegionAccessException is required roles are missing and the
+ * LossAction is either NO_ACCESS or LIMITED_ACCESS.
+ *
+ * @throws RegionAccessException
+ * if required roles are missing and the LossAction is either
+ * NO_ACCESS or LIMITED_ACCESS
+ */
+ @Override
+ protected void checkForLimitedOrNoAccess()
+ {
+ if (this.requiresReliabilityCheck && this.isMissingRequiredRoles) {
+ if (getMembershipAttributes().getLossAction().isNoAccess()
+ || getMembershipAttributes().getLossAction().isLimitedAccess()) {
+ synchronized (this.missingRequiredRoles) {
+ if (!this.isMissingRequiredRoles)
+ return;
+ Set roles = Collections.unmodifiableSet(new HashSet(
+ this.missingRequiredRoles));
+ Assert.assertTrue(!roles.isEmpty());
+ throw new RegionAccessException(LocalizedStrings.DistributedRegion_OPERATION_IS_DISALLOWED_BY_LOSSACTION_0_BECAUSE_THESE_REQUIRED_ROLES_ARE_MISSING_1
+ .toLocalizedString(new Object[] { getMembershipAttributes().getLossAction(), roles}), getFullPath(), roles);
+ }
+ }
+ }
+ }
+
+ @Override
+ protected void handleReliableDistribution(ReliableDistributionData data,
+ Set successfulRecipients) {
+ handleReliableDistribution(data, successfulRecipients,
+ Collections.EMPTY_SET, Collections.EMPTY_SET);
+ }
+
+ protected void handleReliableDistribution(ReliableDistributionData data,
+ Set successfulRecipients, Set otherRecipients1, Set otherRecipients2)
+ {
+ if (this.requiresReliabilityCheck) {
+ MembershipAttributes ra = getMembershipAttributes();
+ Set recipients = successfulRecipients;
+ // determine the successful roles
+ Set roles = new HashSet();
+ for (Iterator iter = recipients.iterator(); iter.hasNext();) {
+ InternalDistributedMember mbr = (InternalDistributedMember)iter.next();
+ if (mbr != null) {
+ roles.addAll(mbr.getRoles());
+ }
+ }
+ for (Iterator iter = otherRecipients1.iterator(); iter.hasNext();) {
+ InternalDistributedMember mbr = (InternalDistributedMember)iter.next();
+ if (mbr != null) {
+ roles.addAll(mbr.getRoles());
+ }
+ }
+ for (Iterator iter = otherRecipients2.iterator(); iter.hasNext();) {
+ InternalDistributedMember mbr = (InternalDistributedMember)iter.next();
+ if (mbr != null) {
+ roles.addAll(mbr.getRoles());
+ }
+ }
+ // determine the missing roles
+ Set failedRoles = new HashSet(ra.getRequiredRoles());
+ failedRoles.removeAll(roles);
+ if (failedRoles.isEmpty())
+ return;
+// if (rp.isAllAccessWithQueuing()) {
+// this.rmq.add(data, failedRoles);
+// } else {
+
+ throw new RegionDistributionException(LocalizedStrings.DistributedRegion_OPERATION_DISTRIBUTION_MAY_HAVE_FAILED_TO_NOTIFY_THESE_REQUIRED_ROLES_0.toLocalizedString(failedRoles), getFullPath(), failedRoles);
+// }
+ }
+ }
+
+ /**
+ *
+ * Called when we do a distributed operation and don't have anyone to
+ * distributed it too. Since this is only called when no distribution was done
+ * (i.e. no recipients) we do not check isMissingRequiredRoles because it
+ * might not longer be true due to race conditions
+ *
+ * @return false if this region has at least one required role and queuing is
+ * configured. Returns true if sending to no one is ok.
+ * @throws RoleException
+ * if a required role is missing and the LossAction is either
+ * NO_ACCESS or LIMITED_ACCESS.
+ * @since 5.0
+ */
+ protected boolean isNoDistributionOk()
+ {
+ if (this.requiresReliabilityCheck) {
+ MembershipAttributes ra = getMembershipAttributes();
+ // if (ra.getLossAction().isAllAccessWithQueuing()) {
+ // return !ra.hasRequiredRoles();
+ // } else {
+ Set failedRoles = ra.getRequiredRoles();
+ throw new RegionDistributionException(LocalizedStrings.DistributedRegion_OPERATION_DISTRIBUTION_WAS_NOT_DONE_TO_THESE_REQUIRED_ROLES_0.toLocalizedString(failedRoles), getFullPath(), failedRoles);
+ // }
+ }
+ return true;
+ }
+
+ /**
+ * returns true if this Region does not distribute its operations to other
+ * members.
+ * @since 6.0
+ * @see HARegion#localDestroyNoCallbacks(Object)
+ */
+ public boolean doesNotDistribute() {
+ return false;
+ }
+
+
+ @Override
+ public boolean shouldSyncForCrashedMember(InternalDistributedMember id) {
+ return !doesNotDistribute() && super.shouldSyncForCrashedMember(id);
+ }
+
+
+ /**
+ * Adjust the specified set of recipients by removing any of them that are
+ * currently having their data queued.
+ *
+ * @param recipients
+ * the set of recipients that a message is to be distributed too.
+ * Recipients that are currently having their data queued will be
+ * removed from this set.
+ * @return the set, possibly null, of recipients that are currently having
+ * their data queued.
+ * @since 5.0
+ */
+ protected Set adjustForQueuing(Set recipients)
+ {
+ Set result = null;
+ // if (this.requiresReliabilityCheck) {
+ // MembershipAttributes ra = getMembershipAttributes();
+ // if (ra.getLossAction().isAllAccessWithQueuing()) {
+ // Set currentQueuedRoles = this.rmq.getQueuingRoles();
+ // if (currentQueuedRoles != null) {
+ // // foreach recipient see if any of his roles are queued and if
+ // // they are remove him from recipients and add him to result
+ // Iterator it = recipients.iterator();
+ // while (it.hasNext()) {
+ // DistributedMember dm = (DistributedMember)it.next();
+ // Set dmRoles = dm.getRoles();
+ // if (!dmRoles.isEmpty()) {
+ // if (intersects(dmRoles, currentQueuedRoles)) {
+ // it.remove(); // fix for bug 34447
+ // if (result == null) {
+ // result = new HashSet();
+ // }
+ // result.add(dm);
+ // }
+ // }
+ // }
+ // }
+ // }
+ // }
+ return result;
+ }
+
+ /**
+ * Returns true if the two sets intersect
+ *
+ * @param a
+ * a non-null non-empty set
+ * @param b
+ * a non-null non-empty set
+ * @return true if sets a and b intersect; false if not
+ * @since 5.0
+ */
+ public static boolean intersects(Set a, Set b)
+ {
+ Iterator it;
+ Set target;
+ if (a.size() <= b.size()) {
+ it = a.iterator();
+ target = b;
+ }
+ else {
+ it = b.iterator();
+ target = a;
+ }
+ while (it.hasNext()) {
+ if (target.contains(it.next()))
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public boolean requiresReliabilityCheck()
+ {
+ return this.requiresReliabilityCheck;
+ }
+
+ /**
+ * Returns true if the ExpiryTask is currently allowed to expire.
+ *
+ * If the region is in NO_ACCESS due to reliability configuration, then no
+ * expiration actions are allowed.
+ *
+ * If the region is in LIMITED_ACCESS due to reliability configuration, then
+ * only non-distributed expiration actions are allowed.
+ */
+ @Override
+ protected boolean isExpirationAllowed(ExpiryTask expiry)
+ {
+ if (this.requiresReliabilityCheck && this.isMissingRequiredRoles) {
+ if (getMembershipAttributes().getLossAction().isNoAccess()) {
+ return false;
+ }
+ if (getMembershipAttributes().getLossAction().isLimitedAccess()
+ && expiry.isDistributedAction()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Performs the resumption action when reliability is resumed.
+ *
+ * @return true if asynchronous resumption is triggered
+ */
+ protected boolean resumeReliability(InternalDistributedMember id,
+ Set newlyAcquiredRoles)
+ {
+ boolean async = false;
+ try {
+ ResumptionAction ra = getMembershipAttributes().getResumptionAction();
+ if (ra.isNone()) {
+ if (logger.isDebugEnabled()) {
+ logger.debug("Reliability resumption for action of none");
+ }
+ resumeExpiration();
+ }
+ else if (ra.isReinitialize()) {
+ async = true;
+ asyncResumeReliability(id, newlyAcquiredRoles);
+ }
+ }
+ catch (Exception e) {
+ logger.fatal(LocalizedMessage.create(LocalizedStrings.DistributedRegion_UNEXPECTED_EXCEPTION), e);
+ }
+ return async;
+ }
+
+ /**
+ * Handles asynchronous ResumptionActions such as region reinitialize.
+ */
+ private void asyncResumeReliability(final InternalDistributedMember id,
+ final Set newlyAcquiredRoles)
+ throws RejectedExecutionException {
+ final ResumptionAction ra = getMembershipAttributes().getResumptionAction();
+ getDistributionManager().getWaitingThreadPool().execute(new Runnable() {
+ public void run()
+ {
+ try {
+ if (ra.isReinitialize()) {
+ if (logger.isDebugEnabled()) {
+ logger.debug("Reliability resumption for action of reinitialize");
+ }
+ if (!isDestroyed() && !cache.isClosed()) {
+ RegionEventImpl event = new RegionEventImpl(
+ DistributedRegion.this, Operation.REGION_REINITIALIZE, null,
+ false, getMyId(), generateEventID());
+ reinitialize(null, event);
+ }
+ synchronized (missingRequiredRoles) {
+ // any number of threads may be waiting on missingRequiredRoles
+ missingRequiredRoles.notifyAll();
+ if (hasListener() && id != null) {
+ // fire afterRoleGain event
+ RoleEventImpl relEvent = new RoleEventImpl(
+ DistributedRegion.this, Operation.REGION_CREATE, null,
+ true, id, newlyAcquiredRoles);
+ dispatchListenerEvent(EnumListenerEvent.AFTER_ROLE_GAIN,
+ relEvent);
+ }
+ }
+ }
+ }
+ catch (Exception e) {
+ logger.fatal(LocalizedMessage.create(LocalizedStrings.DistributedRegion_UNEXPECTED_EXCEPTION), e);
+ }
+ }
+ });
+ }
+
+ /** Reschedules expiry tasks when reliability is resumed. */
+ private void resumeExpiration()
+ {
+ boolean isNoAccess = getMembershipAttributes().getLossAction().isNoAccess();
+ boolean isLimitedAccess = getMembershipAttributes().getLossAction()
+ .isLimitedAccess();
+ if (!(isNoAccess || isLimitedAccess)) {
+ return; // early out: expiration was never affected by reliability
+ }
+
+ if (getEntryTimeToLive().getTimeout() > 0
+ && (isNoAccess || (isLimitedAccess && getEntryTimeToLive().getAction()
+ .isDistributed()))) {
+ rescheduleEntryExpiryTasks();
+ }
+ else
+ if (getEntryIdleTimeout().getTimeout() > 0
+ && (isNoAccess || (isLimitedAccess && getEntryIdleTimeout().getAction()
+ .isDistributed()))) {
+ rescheduleEntryExpiryTasks();
+ }
+ else
+ if (getCustomEntryTimeToLive() != null || getCustomEntryIdleTimeout() != null) {
+ // Force all entries to be rescheduled
+ rescheduleEntryExpiryTasks();
+ }
+
+ if (getRegionTimeToLive().getTimeout() > 0
+ && (isNoAccess || (isLimitedAccess && getRegionTimeToLive().getAction()
+ .isDistributed()))) {
+ addTTLExpiryTask();
+ }
+ if (getRegionIdleTimeout().getTimeout() > 0
+ && (isNoAccess || (isLimitedAccess && getRegionIdleTimeout()
+ .getAction().isDistributed()))) {
+ addIdleExpiryTask();
+ }
+ }
+
+ /**
+ * A boolean used to indicate if its the intialization time i.e the
+ * distributed Region is created for the first time. The variable is used at
+ * the time of lost reliablility.
+ */
+ private boolean isInitializingThread = false;
+
+ /**
+ * Called when reliability is lost. If MembershipAttributes are configured
+ * with {@link LossAction#RECONNECT}then DistributedSystem reconnect will be
+ * called asynchronously.
+ *
+ * @return true if asynchronous resumption is triggered
+ */
+ protected boolean lostReliability(final InternalDistributedMember id,
+ final Set newlyMissingRoles)
+ {
+ if (DistributedRegion.ignoreReconnect)
+ return false;
+ boolean async = false;
+ try {
+ if (getMembershipAttributes().getLossAction().isReconnect()) {
+ async = true;
+ if (isInitializingThread) {
+ doLostReliability(true, id, newlyMissingRoles);
+ }
+ else {
+ doLostReliability(false, id, newlyMissingRoles);
+ }
+ // we don't do this in the waiting pool because we're going to
+ // disconnect
+ // the distributed system, and it will wait for the pool to empty
+ /*
+ * moved to a new method called doLostReliablity. Thread t = new
+ * Thread("Reconnect Distributed System") { public void run() { try { //
+ * TODO: may need to check isReconnecting and checkReadiness...
+ * initializationLatchAfterMemberTimeout.await(); // TODO:
+ * call reconnect here
+ * getSystem().tryReconnect((GemFireCache)getCache()); // added for
+ * reconnect. synchronized (missingRequiredRoles) { // any number of
+ * threads may be waiting on missingRequiredRoles
+ * missingRequiredRoles.notifyAll(); // need to fire an event if id is
+ * not null if (hasListener() && id != null) { RoleEventImpl relEvent =
+ * new RoleEventImpl( DistributedRegion.this, Operation.CACHE_RECONNECT,
+ * null, true, id, newlyMissingRoles); dispatchListenerEvent(
+ * EnumListenerEvent.AFTER_ROLE_LOSS, relEvent); } } } catch (Exception
+ * e) { } } };
+ * t.setDaemon(true); t.start();
+ */
+ }
+ }
+ catch (CancelException cce) {
+ throw cce;
+ }
+ catch (Exception e) {
+ logger.fatal(LocalizedMessage.create(LocalizedStrings.DistributedRegion_UNEXPECTED_EXCEPTION), e);
+ }
+ return async;
+ }
+
+ private void doLostReliability(boolean isInitializing,
+ final InternalDistributedMember id, final Set newlyMissingRoles)
+ {
+ try {
+ if (!isInitializing) {
+ // moved code to a new thread.
+ Thread t = new Thread(LocalizedStrings.DistributedRegion_RECONNECT_DISTRIBUTED_SYSTEM.toLocalizedString()) {
+ @Override
+ public void run()
+ {
+ try {
+ // TODO: may need to check isReconnecting and checkReadiness...
+ if (logger.isDebugEnabled()) {
+ logger.debug("Reliability loss with policy of reconnect and membership thread doing reconnect");
+ }
+ initializationLatchAfterMemberTimeout.await();
+ getSystem().tryReconnect(false, "Role Loss", getCache());
+ synchronized (missingRequiredRoles) {
+ // any number of threads may be waiting on missingRequiredRoles
+ missingRequiredRoles.notifyAll();
+ // need to fire an event if id is not null
+ if (hasListener() && id != null) {
+ RoleEventImpl relEvent = new RoleEventImpl(
+ DistributedRegion.this, Operation.CACHE_RECONNECT, null,
+ true, id, newlyMissingRoles);
+ dispatchListenerEvent(EnumListenerEvent.AFTER_ROLE_LOSS,
+ relEvent);
+ }
+ }
+ }
+ catch (Exception e) {
+ logger.fatal(LocalizedMessage.create(LocalizedStrings.DistributedRegion_UNEXPECTED_EXCEPTION), e);
+ }
+ }
+ };
+ t.setDaemon(true);
+ t.start();
+
+ }
+ else {
+ getSystem().tryReconnect(false, "Role Loss", getCache()); // added for
+ // reconnect.
+ synchronized (missingRequiredRoles) {
+ // any number of threads may be waiting on missingRequiredRoles
+ missingRequiredRoles.notifyAll();
+ // need to fire an event if id is not null
+ if (hasListener() && id != null) {
+ RoleEventImpl relEvent = new RoleEventImpl(DistributedRegion.this,
+ Operation.CACHE_RECONNECT, null, true, id, newlyMissingRoles);
+ dispatchListenerEvent(EnumListenerEvent.AFTER_ROLE_LOSS, relEvent);
+ }
+ }
+ // } catch (CancelException cce){
+
+ // }
+
+ }
+ }
+ catch (CancelException ignor) {
+ throw ignor;
+ }
+ catch (Exception e) {
+ logger.fatal(LocalizedMessage.create(LocalizedStrings.DistributedRegion_UNEXPECTED_EXCEPTION), e);
+ }
+
+ }
+
+ protected void lockCheckReadiness()
+ {
+ // fix for bug 32610
+ cache.getCancelCriterion().checkCancelInProgress(null);
+ checkReadiness();
+ }
+
+ @Override
+ public final Object validatedDestroy(Object key, EntryEventImpl event)
+ throws TimeoutException, EntryNotFoundException, CacheWriterException {
+ Lock dlock = this.getDistributedLockIfGlobal(key);
+ try {
+ return super.validatedDestroy(key, event);
+ } finally {
+ if (dlock != null) {
+ dlock.unlock();
+ }
+ }
+ }
+
+ /**
+ * @see LocalRegion#localDestroyNoCallbacks(Object)
+ */
+ @Override
+ public void localDestroyNoCallbacks(Object key)
+ {
+ super.localDestroyNoCallbacks(key);
+ if (getScope().isGlobal()) {
+ try {
+ this.getLockService().freeResources(key);
+ }
+ catch (LockServiceDestroyedException ignore) {
+ }
+ }
+ }
+
+ /**
+ * @see LocalRegion#localDestroy(Object, Object)
+ */
+ @Override
+ public void localDestroy(Object key, Object aCallbackArgument)
+ throws EntryNotFoundException
+ {
+ super.localDestroy(key, aCallbackArgument);
+ if (getScope().isGlobal()) {
+ try {
+ this.getLockService().freeResources(key);
+ }
+ catch (LockServiceDestroyedException ignore) {
+ }
+ }
+ }
+
+ /**
+ * @see LocalRegion#invalidate(Object, Object)
+ */
+ @Override
+ public void invalidate(Object key, Object aCallbackArgument)
+ throws TimeoutException, EntryNotFoundException
+ {
+ validateKey(key);
+ validateCallbackArg(aCallbackArgument);
+ checkReadiness();
+ checkForLimitedOrNoAccess();
+ Lock dlock = this.getDistributedLockIfGlobal(key);
+ try {
+ super.validatedInvalidate(key, aCallbackArgument);
+ }
+ finally {
+ if (dlock != null)
+ dlock.unlock();
+ }
+ }
+
+ @Override
+ public Lock getRegionDistributedLock() throws IllegalStateException
+ {
+ lockCheckReadiness();
+ checkForLimitedOrNoAccess();
+ if (!this.scope.isGlobal()) {
+ throw new IllegalStateException(LocalizedStrings.DistributedRegion_DISTRIBUTION_LOCKS_ARE_ONLY_SUPPORTED_FOR_REGIONS_WITH_GLOBAL_SCOPE_NOT_0.toLocalizedString(this.scope));
+ }
+ return new RegionDistributedLock();
+ }
+
+ @Override
+ public Lock getDistributedLock(Object key) throws IllegalStateException
+ {
+ validateKey(key);
+ lockCheckReadiness();
+ checkForLimitedOrNoAccess();
+ if (!this.scope.isGlobal()) {
+ throw new IllegalStateException(LocalizedStrings.DistributedRegion_DISTRIBUTION_LOCKS_ARE_ONLY_SUPPORTED_FOR_REGIONS_WITH_GLOBAL_SCOPE_NOT_0.toLocalizedString(this.scope));
+ }
+ if (isLockingSuspendedByCurrentThread()) {
+ throw new IllegalStateException(LocalizedStrings.DistributedRegion_THIS_THREAD_HAS_SUSPENDED_ALL_LOCKING_FOR_THIS_REGION.toLocalizedString());
+ }
+ return new DistributedLock(key);
+ }
+
+ /**
+ * Called while NOT holding lock on parent's subregions
+ *
+ * @throws IllegalStateException
+ * if region is not compatible with a region in another VM.
+ *
+ * @see LocalRegion#initialize(InputStream, InternalDistributedMember, InternalRegionArguments)
+ */
+ @Override
+ protected void initialize(InputStream snapshotInputStream,
+ InternalDistributedMember imageTarget, InternalRegionArguments internalRegionArgs) throws TimeoutException,
+ IOException, ClassNotFoundException
+ {
+ Assert.assertTrue(!isInitialized());
+ if (logger.isDebugEnabled()) {
+ logger.debug("DistributedRegion.initialize BEGIN: {}", getFullPath());
+ }
+
+ // if we're versioning entries we need a region-level version vector
+ if (this.scope.isDistributed() && this.concurrencyChecksEnabled) {
+ createVersionVector();
+ }
+
+ if (this.scope.isGlobal()) {
+ getLockService(); // create lock service eagerly now
+ }
+
+ final IndexUpdater indexUpdater = getIndexUpdater();
+ boolean sqlfGIILockTaken = false;
+ // this try block is to release the SQLF GII lock in finally
+ // which should be done after bucket status will be set
+ // properly in LocalRegion#initialize()
+ try {
+ try {
+ try {
+ // take the GII lock to avoid missing entries while updating the
+ // index list for SQLFabric (#41330 and others)
+ if (indexUpdater != null) {
+ indexUpdater.lockForGII();
+ sqlfGIILockTaken = true;
+ }
+
+ PersistentMemberID persistentId = null;
+ boolean recoverFromDisk = isRecoveryNeeded();
+ DiskRegion dskRgn = getDiskRegion();
+ if (recoverFromDisk) {
+ if (logger.isDebugEnabled()) {
+ logger.debug("DistributedRegion.getInitialImageAndRecovery: Starting Recovery");
+ }
+ dskRgn.initializeOwner(this); // do recovery
+ if (logger.isDebugEnabled()) {
+ logger.debug("DistributedRegion.getInitialImageAndRecovery: Finished Recovery");
+ }
+ persistentId = dskRgn.getMyPersistentID();
+ }
+
+ // Create OQL indexes before starting GII.
+ createOQLIndexes(internalRegionArgs, recoverFromDisk);
+
+ if (getDataPolicy().withReplication()
+ || getDataPolicy().withPreloaded()) {
+ getInitialImageAndRecovery(snapshotInputStream, imageTarget,
+ internalRegionArgs, recoverFromDisk, persistentId);
+ }
+ else {
+ new CreateRegionProcessor(this).initializeRegion();
+ if (snapshotInputStream != null) {
+ releaseBeforeGetInitialImageLatch();
+ loadSnapshotDuringInitialization(snapshotInputStream);
+ }
+ }
+ }
+ catch (DiskAccessException dae) {
+ this.handleDiskAccessException(dae, true);
+ throw dae;
+ }
+
+ initMembershipRoles();
+ isInitializingThread = false;
+ super.initialize(null, null, null); // makes sure all latches are released if they haven't been already
+ } finally {
+ if (this.eventTracker != null) {
+ this.eventTracker.setInitialized();
+ }
+ }
+ } finally {
+ if (sqlfGIILockTaken) {
+ indexUpdater.unlockForGII();
+ }
+ }
+ }
+
+ @Override
+ public void initialized() {
+ new UpdateAttributesProcessor(this).distribute(false);
+ }
+
+ /** True if GII was impacted by missing required roles */
+ private boolean giiMissingRequiredRoles = false;
+
+ /**
+ * A reference counter to protected the memoryThresholdReached boolean
+ */
+ private final SetMembershipListeners that are registered on this
+ * manager. */
+ private final ConcurrentMap membershipListeners;
+
+ /** A lock to hold while adding and removing membership listeners */
+ protected final Object membershipListenersLock =
+ new MembershipListenersLock();
+ /** The MembershipListeners that are registered on this
+ * manager for ALL members.
+ * @since 5.7
+ */
+ protected volatile Set allMembershipListeners = Collections.EMPTY_SET;
+ /**
+ * A lock to hold while adding and removing all membership listeners.
+ * @since 5.7
+ */
+ protected final Object allMembershipListenersLock =
+ new MembershipListenersLock();
+ /** A queue of MemberEvent instances */
+ protected final BlockingQueue membershipEventQueue =
+ new LinkedBlockingQueue();
+ /** Used to invoke registered membership listeners in the background. */
+ private Thread memberEventThread;
+
+
+ /** A brief description of this DistributionManager */
+ protected final String description;
+
+ /** Statistics about distribution */
+ protected /*final*/ DistributionStats stats;
+
+ /** Did an exception occur in one of the DM threads? */
+ protected boolean exceptionInThreads;
+
+ static ThreadLocal isStartupThread = new ThreadLocal();
+
+ protected volatile boolean shutdownMsgSent = false;
+
+ /** Set to true when this manager is being shutdown */
+ protected volatile boolean closeInProgress = false;
+
+ private volatile boolean receivedStartupResponse = false;
+
+ private volatile String rejectionMessage = null;
+
+ protected MembershipManager membershipManager;
+
+ /** The channel through which distributed communication occurs. */
+ protected DistributionChannel channel;
+
+ /**
+ * The (non-admin-only) members of the distributed system. This is a
+ * map of memberid->memberid for fast access to canonical ID references.
+ * All accesses to this
+ * field must be synchronized on {@link #membersLock}.
+ */
+ private Map
+ * 1) {@link #members}
+ * 2) {@link #membersAndAdmin}
+ * 3) {@link #hostedLocatorsAll}
+ * 4) {@link #hostedLocatorsWithSharedConfiguration}
+ */
+ private final Object membersLock = new MembersLock();
+
+ /**
+ * The lock held while writing {@link #adminConsoles}.
+ */
+ private final Object adminConsolesLock = new Object();
+ /**
+ * The ids of all known admin consoles
+ * Uses Copy on Write. Writers must sync on adminConsolesLock.
+ * Readers don't need to sync.
+ */
+ private volatile SetDistributionManager by initializing
+ * itself, creating the membership manager and executors
+ *
+ * @param transport
+ * The configuration for the communications transport
+ *
+ */
+ private DistributionManager(RemoteTransportConfig transport,
+ InternalDistributedSystem system) {
+
+ this.dmType = transport.getVmKind();
+ this.system = system;
+ this.elderLock = new StoppableReentrantLock(stopper);
+ this.transport = transport;
+
+ this.membershipListeners = new ConcurrentHashMap();
+ this.distributedSystemId = system.getConfig().getDistributedSystemId();
+ {
+ long statId = OSProcess.getId();
+ /* deadcoded since we don't know the channel id yet.
+ if (statId == 0 || statId == -1) {
+ statId = getChannelId();
+ }
+ */
+ this.stats = new DistributionStats(system, statId);
+ DistributionStats.enableClockStats = system.getConfig().getEnableTimeStatistics();
+ }
+
+ this.exceptionInThreads = false;
+
+ // Start the processing threads
+ final LoggingThreadGroup group =
+ LoggingThreadGroup.createThreadGroup("DistributionManager Threads", logger);
+ this.threadGroup = group;
+
+ boolean finishedConstructor = false;
+ try {
+
+ if (MULTI_SERIAL_EXECUTORS) {
+ if (logger.isInfoEnabled(LogMarker.DM)) {
+ logger.info(LogMarker.DM,
+ "Serial Queue info :" +
+ " THROTTLE_PERCENT: " + THROTTLE_PERCENT +
+ " SERIAL_QUEUE_BYTE_LIMIT :" + SERIAL_QUEUE_BYTE_LIMIT +
+ " SERIAL_QUEUE_THROTTLE :" + SERIAL_QUEUE_THROTTLE +
+ " TOTAL_SERIAL_QUEUE_BYTE_LIMIT :" + TOTAL_SERIAL_QUEUE_BYTE_LIMIT +
+ " TOTAL_SERIAL_QUEUE_THROTTLE :" + TOTAL_SERIAL_QUEUE_THROTTLE +
+ " SERIAL_QUEUE_SIZE_LIMIT :" + SERIAL_QUEUE_SIZE_LIMIT +
+ " SERIAL_QUEUE_SIZE_THROTTLE :" + SERIAL_QUEUE_SIZE_THROTTLE
+ );
+ }
+ // when TCP/IP is disabled we can't throttle the serial queue or we run the risk of
+ // distributed deadlock when we block the UDP reader thread
+ boolean throttlingDisabled = system.getConfig().getDisableTcp();
+ this.serialQueuedExecutorPool = new SerialQueuedExecutorPool(this.threadGroup, this.stats, throttlingDisabled);
+ }
+
+ {
+ BlockingQueue poolQueue;
+ if (SERIAL_QUEUE_BYTE_LIMIT == 0) {
+ poolQueue = new OverflowQueueWithDMStats(this.stats.getSerialQueueHelper());
+ } else {
+ this.serialQueue = new ThrottlingMemLinkedQueueWithDMStats(TOTAL_SERIAL_QUEUE_BYTE_LIMIT,
+ TOTAL_SERIAL_QUEUE_THROTTLE, SERIAL_QUEUE_SIZE_LIMIT, SERIAL_QUEUE_SIZE_THROTTLE,
+ this.stats.getSerialQueueHelper());
+ poolQueue = this.serialQueue;
+ }
+ ThreadFactory tf = new ThreadFactory() {
+ public Thread newThread(final Runnable command) {
+ DistributionManager.this.stats.incSerialThreadStarts();
+ final Runnable r = new Runnable() {
+ public void run() {
+ DistributionManager.this.stats.incNumSerialThreads(1);
+ try {
+ ConnectionTable.threadWantsSharedResources();
+ Connection.makeReaderThread();
+ runUntilShutdown(command);
+ // command.run();
+ } finally {
+ ConnectionTable.releaseThreadsSockets();
+ DistributionManager.this.stats.incNumSerialThreads(-1);
+ }
+ }
+ };
+ Thread thread = new Thread(group, r, LocalizedStrings.DistributionManager_SERIAL_MESSAGE_PROCESSOR.toLocalizedString());
+ thread.setDaemon(true);
+ return thread;
+ }
+ };
+ SerialQueuedExecutorWithDMStats executor = new SerialQueuedExecutorWithDMStats(poolQueue,
+ this.stats.getSerialProcessorHelper(), tf);
+ this.serialThread = executor;
+ }
+ {
+ BlockingQueue q = new LinkedBlockingQueue();
+ ThreadFactory tf = new ThreadFactory() {
+ public Thread newThread(final Runnable command) {
+ DistributionManager.this.stats.incViewThreadStarts();
+ final Runnable r = new Runnable() {
+ public void run() {
+ DistributionManager.this.stats.incNumViewThreads(1);
+ try {
+ ConnectionTable.threadWantsSharedResources();
+ Connection.makeReaderThread();
+ runUntilShutdown(command);
+ } finally {
+ ConnectionTable.releaseThreadsSockets();
+ DistributionManager.this.stats.incNumViewThreads(-1);
+ }
+ }
+ };
+ Thread thread = new Thread(group, r, LocalizedStrings.DistributionManager_VIEW_MESSAGE_PROCESSOR.toLocalizedString());
+ thread.setDaemon(true);
+ return thread;
+ }
+ };
+ this.viewThread = new SerialQueuedExecutorWithDMStats(q,
+ this.stats.getViewProcessorHelper(), tf);
+ }
+
+ {
+ BlockingQueue poolQueue;
+ if (INCOMING_QUEUE_LIMIT == 0) {
+ poolQueue = new OverflowQueueWithDMStats(this.stats.getOverflowQueueHelper());
+ } else {
+ poolQueue = new OverflowQueueWithDMStats(INCOMING_QUEUE_LIMIT, this.stats.getOverflowQueueHelper());
+ }
+ ThreadFactory tf = new ThreadFactory() {
+ private int next = 0;
+
+ public Thread newThread(final Runnable command) {
+ DistributionManager.this.stats.incProcessingThreadStarts();
+ final Runnable r = new Runnable() {
+ public void run() {
+ DistributionManager.this.stats.incNumProcessingThreads(1);
+ try {
+ ConnectionTable.threadWantsSharedResources();
+ Connection.makeReaderThread();
+ runUntilShutdown(command);
+ } finally {
+ ConnectionTable.releaseThreadsSockets();
+ DistributionManager.this.stats.incNumProcessingThreads(-1);
+ }
+ }
+ };
+ Thread thread = new Thread(group, r,
+ LocalizedStrings.DistributionManager_POOLED_MESSAGE_PROCESSOR.toLocalizedString() + (next++));
+ thread.setDaemon(true);
+ return thread;
+ }
+ };
+ ThreadPoolExecutor pool =
+ new PooledExecutorWithDMStats(poolQueue, MAX_THREADS, this.stats.getNormalPoolHelper(), tf);
+ this.threadPool = pool;
+ }
+
+
+ {
+ BlockingQueue poolQueue;
+ if (INCOMING_QUEUE_LIMIT == 0) {
+ poolQueue = new OverflowQueueWithDMStats(this.stats.getHighPriorityQueueHelper());
+ } else {
+ poolQueue = new OverflowQueueWithDMStats(INCOMING_QUEUE_LIMIT, this.stats.getHighPriorityQueueHelper());
+ }
+ ThreadFactory tf = new ThreadFactory() {
+ private int next = 0;
+
+ public Thread newThread(final Runnable command) {
+ DistributionManager.this.stats.incHighPriorityThreadStarts();
+ final Runnable r = new Runnable() {
+ public void run() {
+ DistributionManager.this.stats.incHighPriorityThreads(1);
+ try {
+ ConnectionTable.threadWantsSharedResources();
+ Connection.makeReaderThread();
+ runUntilShutdown(command);
+ } finally {
+ ConnectionTable.releaseThreadsSockets();
+ DistributionManager.this.stats.incHighPriorityThreads(-1);
+ }
+ }
+ };
+ Thread thread = new Thread(group, r,
+ LocalizedStrings.DistributionManager_POOLED_HIGH_PRIORITY_MESSAGE_PROCESSOR.toLocalizedString() + (next++));
+ thread.setDaemon(true);
+ return thread;
+ }
+ };
+ this.highPriorityPool = new PooledExecutorWithDMStats(poolQueue, MAX_THREADS, this.stats.getHighPriorityPoolHelper(), tf);
+ }
+
+
+ {
+ ThreadFactory tf = new ThreadFactory() {
+ private int next = 0;
+
+ public Thread newThread(final Runnable command) {
+ DistributionManager.this.stats.incWaitingThreadStarts();
+ final Runnable r = new Runnable() {
+ public void run() {
+ DistributionManager.this.stats.incWaitingThreads(1);
+ try {
+ ConnectionTable.threadWantsSharedResources();
+ Connection.makeReaderThread();
+ runUntilShutdown(command);
+ } finally {
+ ConnectionTable.releaseThreadsSockets();
+ DistributionManager.this.stats.incWaitingThreads(-1);
+ }
+ }
+ };
+ Thread thread = new Thread(group, r,
+ LocalizedStrings.DistributionManager_POOLED_WAITING_MESSAGE_PROCESSOR.toLocalizedString() + (next++));
+ thread.setDaemon(true);
+ return thread;
+ }
+ };
+ BlockingQueue poolQueue;
+ if (MAX_WAITING_THREADS == Integer.MAX_VALUE) {
+ // no need for a queue since we have infinite threads
+ poolQueue = new SynchronousQueue();
+ } else {
+ poolQueue = new OverflowQueueWithDMStats(this.stats.getWaitingQueueHelper());
+ }
+ this.waitingPool = new PooledExecutorWithDMStats(poolQueue,
+ MAX_WAITING_THREADS,
+ this.stats.getWaitingPoolHelper(),
+ tf);
+ }
+
+ {
+ ThreadFactory tf = new ThreadFactory() {
+ private int next = 0;
+
+ public Thread newThread(final Runnable command) {
+ DistributionManager.this.stats.incWaitingThreadStarts();//will it be ok?
+ final Runnable r = new Runnable() {
+ public void run() {
+ DistributionManager.this.stats.incWaitingThreads(1);//will it be ok
+ try {
+ ConnectionTable.threadWantsSharedResources();
+ Connection.makeReaderThread();
+ runUntilShutdown(command);
+ } finally {
+ ConnectionTable.releaseThreadsSockets();
+ DistributionManager.this.stats.incWaitingThreads(-1);
+ }
+ }
+ };
+ Thread thread = new Thread(group, r,
+ LocalizedStrings.DistributionManager_PR_META_DATA_CLEANUP_MESSAGE_PROCESSOR.toLocalizedString() + (next++));
+ thread.setDaemon(true);
+ return thread;
+ }
+ };
+ BlockingQueue poolQueue;
+ poolQueue = new OverflowQueueWithDMStats(this.stats.getWaitingQueueHelper());
+ this.prMetaDataCleanupThreadPool = new PooledExecutorWithDMStats(poolQueue,
+ MAX_PR_META_DATA_CLEANUP_THREADS,
+ this.stats.getWaitingPoolHelper(),
+ tf);
+ }
+
+ {
+ BlockingQueue poolQueue;
+ if (INCOMING_QUEUE_LIMIT == 0) {
+ poolQueue = new OverflowQueueWithDMStats(this.stats.getPartitionedRegionQueueHelper());
+ } else {
+ poolQueue = new OverflowQueueWithDMStats(INCOMING_QUEUE_LIMIT, this.stats.getPartitionedRegionQueueHelper());
+ }
+ ThreadFactory tf = new ThreadFactory() {
+ private int next = 0;
+
+ public Thread newThread(final Runnable command) {
+ DistributionManager.this.stats.incPartitionedRegionThreadStarts();
+ final Runnable r = new Runnable() {
+ public void run() {
+ stats.incPartitionedRegionThreads(1);
+ try {
+ ConnectionTable.threadWantsSharedResources();
+ Connection.makeReaderThread();
+ runUntilShutdown(command);
+ } finally {
+ ConnectionTable.releaseThreadsSockets();
+ stats.incPartitionedRegionThreads(-1);
+ }
+ }
+ };
+ Thread thread = new Thread(group, r,
+ "PartitionedRegion Message Processor" + (next++));
+ thread.setDaemon(true);
+ return thread;
+ }
+ };
+ if (MAX_PR_THREADS > 1) {
+ this.partitionedRegionPool = new PooledExecutorWithDMStats(poolQueue,
+ MAX_PR_THREADS, this.stats.getPartitionedRegionPoolHelper(), tf);
+ } else {
+ SerialQueuedExecutorWithDMStats executor = new SerialQueuedExecutorWithDMStats(poolQueue,
+ this.stats.getPartitionedRegionPoolHelper(), tf);
+ this.partitionedRegionThread = executor;
+ }
+
+ }
+
+ {
+ BlockingQueue poolQueue;
+ if (INCOMING_QUEUE_LIMIT == 0) {
+ poolQueue = new OverflowQueueWithDMStats(this.stats.getFunctionExecutionQueueHelper());
+ } else {
+ poolQueue = new OverflowQueueWithDMStats(INCOMING_QUEUE_LIMIT, this.stats.getFunctionExecutionQueueHelper());
+ }
+ ThreadFactory tf = new ThreadFactory() {
+ private int next = 0;
+
+ public Thread newThread(final Runnable command) {
+ DistributionManager.this.stats.incFunctionExecutionThreadStarts();
+ final Runnable r = new Runnable() {
+ public void run() {
+ stats.incFunctionExecutionThreads(1);
+ isFunctionExecutionThread.set(Boolean.TRUE);
+ try {
+ ConnectionTable.threadWantsSharedResources();
+ Connection.makeReaderThread();
+ runUntilShutdown(command);
+ } finally {
+ ConnectionTable.releaseThreadsSockets();
+ stats.incFunctionExecutionThreads(-1);
+ }
+ }
+ };
+ Thread thread = new Thread(group, r,
+ "Function Execution Processor" + (next++));
+ thread.setDaemon(true);
+ return thread;
+ }
+ };
+
+ if(MAX_FE_THREADS > 1){
+ this.functionExecutionPool = new FunctionExecutionPooledExecutor(poolQueue,
+ MAX_FE_THREADS, this.stats.getFunctionExecutionPoolHelper(), tf,true /*for fn exec*/);
+ } else {
+ SerialQueuedExecutorWithDMStats executor = new SerialQueuedExecutorWithDMStats(poolQueue,
+ this.stats.getFunctionExecutionPoolHelper(), tf);
+ this.functionExecutionThread = executor;
+ }
+
+ }
+
+ if (!SYNC_EVENTS) {
+ this.memberEventThread = new Thread(group, new MemberEventInvoker(),
+ "DM-MemberEventInvoker");
+ this.memberEventThread.setDaemon(true);
+ }
+
+ StringBuffer sb = new StringBuffer(" (took ");
+
+ long start = System.currentTimeMillis();
+
+ // Create direct channel first
+// DirectChannel dc = new DirectChannel(new MyListener(this), system.getConfig(), logger, null);
+// setDirectChannelPort(dc.getPort()); // store in a thread local
+
+ // connect to JGroups
+ start = System.currentTimeMillis();
+
+ MyListener l = new MyListener(this);
+ membershipManager = MemberFactory.newMembershipManager(l, system.getConfig(), transport, stats);
+
+ sb.append(System.currentTimeMillis() - start);
+
+ this.myid = membershipManager.getLocalMember();
+
+// dc.patchUpAddress(this.myid);
+// id.setDirectChannelPort(dc.getPort());
+
+ // create the distribution channel
+ this.channel = new DistributionChannel(membershipManager);
+
+ membershipManager.postConnect();
+
+ //Assert.assertTrue(this.getChannelMap().size() >= 1);
+ // System.out.println("Channel Map:");
+ // for (Iterator iter = this.getChannelMap().entrySet().iterator();
+ // iter.hasNext(); ) {
+ // Map.Entry entry = (Map.Entry) iter.next();
+ // Object key = entry.getKey();
+ // System.out.println(" " + key + " a " +
+ // key.getClass().getName() + " -> " +
+ // entry.getValue());
+ // }
+
+ sb.append(" ms)");
+
+ logger.info(LocalizedMessage.create(LocalizedStrings.DistributionManager_STARTING_DISTRIBUTIONMANAGER_0_1,
+ new Object[] { this.myid, (logger.isInfoEnabled(LogMarker.DM) ? sb.toString() : "")}));
+
+ this.description = NAME + " on " + this.myid + " started at "
+ + (new Date(System.currentTimeMillis())).toString();
+
+ finishedConstructor = true;
+ } finally {
+ if (!finishedConstructor) {
+ askThreadsToStop(); // fix for bug 42039
+ }
+ }
+ }
+
+ /**
+ * Creates a new distribution manager
+ *
+ * @param system
+ * The distributed system to which this distribution manager
+ * will send messages.
+ */
+ private DistributionManager(
+ InternalDistributedSystem system,
+ RemoteTransportConfig transport)
+ {
+ this(transport, system);
+
+ boolean finishedConstructor = false;
+ try {
+
+ isStartupThread.set(Boolean.TRUE);
+
+ startThreads();
+
+ // Since we need a StartupResponseMessage to make sure licenses
+ // are compatible the following has been deadcoded.
+// // For the time being, invoke processStartupResponse()
+// String rejectionMessage = null;
+// if (GemFireVersion.getGemFireVersion().
+// equals(state.getGemFireVersion())) {
+// rejectionMessage = "Rejected new system node " +
+// this.getDistributionManagerId() + " with version \"" +
+// GemFireVersion.getGemFireVersion() +
+// "\" because the distributed system's version is \"" +
+// state.getGemFireVersion() + "\".";
+// }
+// this.processStartupResponse(state.getCacheTime(),
+// rejectionMessage);
+
+ // Allow events to start being processed.
+ membershipManager.startEventProcessing();
+ for (;;) {
+ this.getCancelCriterion().checkCancelInProgress(null);
+ boolean interrupted = Thread.interrupted();
+ try {
+ membershipManager.waitForEventProcessing();
+ break;
+ }
+ catch (InterruptedException e) {
+ interrupted = true;
+ }
+ finally {
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ synchronized (DistributionManager.class) {
+ openDMs++;
+ }
+ finishedConstructor = true;
+ } finally {
+ if (!finishedConstructor) {
+ askThreadsToStop(); // fix for bug 42039
+ }
+ }
+ }
+
+ //////////////////// Instance Methods /////////////////////
+
+ /**
+ * Returns true if the two members are on the same equivalent host based
+ * on overlapping IP addresses collected for all NICs during exchange of
+ * startup messages.
+ *
+ * @param member1 First member
+ * @param member2 Second member
+ */
+ public boolean areOnEquivalentHost(InternalDistributedMember member1,
+ InternalDistributedMember member2) {
+ SetDistributionManager
+ */
+ public InternalDistributedMember getId() {
+ return this.myid;
+ }
+
+ /**
+ * Returns the id of the underlying distribution channel used for
+ * communication.
+ *
+ * @since 3.0
+ */
+ public long getChannelId() {
+ return this.channel.getId();
+ }
+
+ /**
+ * Adds a message to the outgoing queue. Note that
+ * message should not be modified after it has been
+ * added to the queue. After message is distributed,
+ * it will be recycled.
+ *
+ * @return list of recipients who did not receive the message
+ * @throws NotSerializableException if the content is not serializable
+ */
+ public Set putOutgoingUserData(final DistributionMessage message)
+ throws NotSerializableException {
+ return sendMessage(message);
+ }
+
+ /**
+ * Send outgoing data; message is guaranteed to be serialized.
+ * @return list of recipients who did not receive the message
+ * @throws InternalGemFireException if message is not serializable
+ */
+ public Set putOutgoing(final DistributionMessage msg) {
+ try {
+ DistributionMessageObserver observer = DistributionMessageObserver.getInstance();
+ if(observer != null) {
+ observer.beforeSendMessage(this, msg);
+ }
+ return sendMessage(msg);
+ }
+ catch (NotSerializableException e) {
+ throw new InternalGemFireException(e);
+ }
+ catch (ToDataException e) {
+ // exception from user code
+ throw e;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return this.description;
+ }
+
+ /**
+ * @see #closeInProgress
+ */
+ private final Object shutdownMutex = new Object();
+
+ /**
+ * Informs other members that this dm is shutting down.
+ * Stops the pusher, puller, and processor threads and closes the
+ * connection to the transport layer.
+ */
+ protected void shutdown() {
+ // Make sure only one thread initiates shutdown...
+ synchronized (shutdownMutex) {
+ if (closeInProgress) {
+ return;
+ }
+ this.closeInProgress = true;
+ } // synchronized
+
+ // [bruce] log shutdown at info level and with ID to balance the
+ // "Starting" message. recycleConn.conf is hard to debug w/o this
+ final String exceptionStatus = (this.exceptionInThreads() ? LocalizedStrings.DistributionManager_AT_LEAST_ONE_EXCEPTION_OCCURRED.toLocalizedString() : "");
+ logger.info(LocalizedMessage.create(
+ LocalizedStrings.DistributionManager_SHUTTING_DOWN_DISTRIBUTIONMANAGER_0_1,
+ new Object[] {this.myid, exceptionStatus}));
+
+ final long start = System.currentTimeMillis();
+ try {
+ if (this.rootCause instanceof ForcedDisconnectException) {
+ if (logger.isDebugEnabled()) {
+ logger.debug("inhibiting sending of shutdown message to other members due to forced-disconnect");
+ }
+ } else {
+ // Don't block indefinitely trying to send the shutdown message, in
+ // case other VMs in the system are ill-behaved. (bug 34710)
+ final Runnable r = new Runnable() {
+ public void run() {
+ try {
+ ConnectionTable.threadWantsSharedResources();
+ sendShutdownMessage();
+ }
+ catch (final CancelException e) {
+ // We were terminated.
+ logger.debug("Cancelled during shutdown message", e);
+ }
+ }
+ };
+ final Thread t = new Thread(threadGroup,
+ r, LocalizedStrings.DistributionManager_SHUTDOWN_MESSAGE_THREAD_FOR_0.toLocalizedString(this.myid));
+ t.start();
+ boolean interrupted = Thread.interrupted();
+ try {
+ t.join(MAX_STOP_TIME);
+ }
+ catch (final InterruptedException e) {
+ interrupted = true;
+ t.interrupt();
+ logger.warn(LocalizedMessage.create(
+ LocalizedStrings.DistributionManager_INTERRUPTED_SENDING_SHUTDOWN_MESSAGE_TO_PEERS), e);
+ }
+ finally {
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ if (t.isAlive()) {
+ t.interrupt();
+ logger.warn(LocalizedMessage.create(LocalizedStrings.DistributionManager_FAILED_SENDING_SHUTDOWN_MESSAGE_TO_PEERS_TIMEOUT));
+ }
+ }
+
+ }
+ finally {
+ this.shutdownMsgSent = true; // in case sendShutdownMessage failed....
+ try {
+ this.uncleanShutdown(false);
+ }
+ finally {
+ final Long delta = Long.valueOf(System.currentTimeMillis() - start);
+ logger.info(LocalizedMessage.create(
+ LocalizedStrings.DistributionManager_DISTRIBUTIONMANAGER_STOPPED_IN_0_MS, delta));
+ }
+ }
+ }
+
+ private void askThreadsToStop() {
+ // Stop executors after they have finished
+ ExecutorService es;
+ es = this.serialThread;
+ if (es != null) {
+ es.shutdown();
+ }
+ es = this.viewThread;
+ if (es != null) {
+ // Hmmm...OK, I'll let any view events currently in the queue be
+ // processed. Not sure it's very important whether they get
+ // handled...
+ es.shutdown();
+ }
+ if (this.serialQueuedExecutorPool != null) {
+ this.serialQueuedExecutorPool.shutdown();
+ }
+ es = this.functionExecutionThread;
+ if (es != null) {
+ es.shutdown();
+ }
+ es = this.functionExecutionPool;
+ if (es != null) {
+ es.shutdown();
+ }
+ es = this.partitionedRegionThread;
+ if (es != null) {
+ es.shutdown();
+ }
+ es = this.partitionedRegionPool;
+ if (es != null) {
+ es.shutdown();
+ }
+ es = this.highPriorityPool;
+ if (es != null) {
+ es.shutdown();
+ }
+ es = this.waitingPool;
+ if (es != null) {
+ es.shutdown();
+ }
+ es = this.prMetaDataCleanupThreadPool;
+ if (es != null) {
+ es.shutdown();
+ }
+ es = this.threadPool;
+ if (es != null) {
+ es.shutdown();
+ }
+
+ Thread th = this.memberEventThread;
+ if (th != null)
+ th.interrupt();
+ }
+
+ private void waitForThreadsToStop(long timeInMillis) throws InterruptedException {
+ long start = System.currentTimeMillis();
+ long remaining = timeInMillis;
+
+ ExecutorService[] allExecutors = new ExecutorService[] {
+ this.serialThread,
+ this.viewThread,
+ this.functionExecutionThread,
+ this.functionExecutionPool,
+ this.partitionedRegionThread,
+ this.partitionedRegionPool,
+ this.highPriorityPool,
+ this.waitingPool,
+ this.prMetaDataCleanupThreadPool,
+ this.threadPool};
+ for(ExecutorService es : allExecutors) {
+ if (es != null) {
+ es.awaitTermination(remaining, TimeUnit.MILLISECONDS);
+ }
+ remaining = timeInMillis - (System.currentTimeMillis() - start);
+ if(remaining <= 0) {
+ return;
+ }
+ }
+
+
+ this.serialQueuedExecutorPool.awaitTermination(remaining, TimeUnit.MILLISECONDS);
+ remaining = timeInMillis - (System.currentTimeMillis() - start);
+ if(remaining <= 0) {
+ return;
+ }
+ Thread th = this.memberEventThread;
+ if (th != null) {
+ th.interrupt(); // bug #43452 - this thread sometimes eats interrupts, so we interrupt it again here
+ th.join(remaining);
+ }
+
+ }
+
+ /**
+ * maximum time, in milliseconds, to wait for all threads to exit
+ */
+ static private final int MAX_STOP_TIME = 20000;
+
+ /**
+ * Time to sleep, in milliseconds, while polling to see if threads have
+ * finished
+ */
+ static private final int STOP_PAUSE_TIME = 1000;
+
+ /**
+ * Maximum number of interrupt attempts to stop a thread
+ */
+ static private final int MAX_STOP_ATTEMPTS = 10;
+
+ /**
+ * Cheap tool to kill a referenced thread
+ *
+ * @param t the thread to kill
+ */
+ private void clobberThread(Thread t) {
+ if (t == null)
+ return;
+ if (t.isAlive()) {
+ logger.warn(LocalizedMessage.create(LocalizedStrings.DistributionManager_FORCING_THREAD_STOP_ON__0_, t));
+
+ // Start by being nice.
+ t.interrupt();
+
+// we could be more violent here...
+// t.stop();
+ try {
+ for (int i = 0; i < MAX_STOP_ATTEMPTS && t.isAlive(); i++) {
+ t.join(STOP_PAUSE_TIME);
+ t.interrupt();
+ }
+ }
+ catch (InterruptedException ex) {
+ logger.debug("Interrupted while attempting to terminate threads.");
+ Thread.currentThread().interrupt();
+ // just keep going
+ }
+
+ if (t.isAlive()) {
+ logger.warn(LocalizedMessage.create(LocalizedStrings.DistributionManager_CLOBBERTHREAD_THREAD_REFUSED_TO_DIE__0, t));
+ }
+ }
+ }
+
+ /**
+ * Cheap tool to examine an executor to see if it is still working
+ * @param tpe
+ * @return true if executor is still active
+ */
+ private boolean executorAlive(ThreadPoolExecutor tpe, String name)
+ {
+ if (tpe == null) {
+ return false;
+ } else {
+ int ac = tpe.getActiveCount();
+// boolean result = tpe.getActiveCount() > 0;
+ if (ac > 0) {
+ if (logger.isDebugEnabled()) {
+ logger.debug("Still waiting for {} threads in '{}' pool to exit", ac, name);
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+ }
+
+ /**
+ * Wait for the ancillary queues to exit. Kills them if they are
+ * still around.
+ *
+ */
+ private void forceThreadsToStop() {
+ long endTime = System.currentTimeMillis() + MAX_STOP_TIME;
+ String culprits = "";
+ for (;;) {
+ boolean stillAlive = false;
+ culprits = "";
+ if (executorAlive(this.serialThread, "serial thread")) {
+ stillAlive = true;
+ culprits = culprits + " serial thread;";
+ }
+ if (executorAlive(this.viewThread, "view thread")) {
+ stillAlive = true;
+ culprits = culprits + " view thread;";
+ }
+ if (executorAlive(this.partitionedRegionThread, "partitioned region thread")) {
+ stillAlive = true;
+ culprits = culprits + " partitioned region thread;";
+ }
+ if (executorAlive(this.partitionedRegionPool, "partitioned region pool")) {
+ stillAlive = true;
+ culprits = culprits + " partitioned region pool;";
+ }
+ if (executorAlive(this.highPriorityPool, "high priority pool")) {
+ stillAlive = true;
+ culprits = culprits + " high priority pool;";
+ }
+ if (executorAlive(this.waitingPool, "waiting pool")) {
+ stillAlive = true;
+ culprits = culprits + " waiting pool;";
+ }
+ if (executorAlive(this.prMetaDataCleanupThreadPool, "prMetaDataCleanupThreadPool")) {
+ stillAlive = true;
+ culprits = culprits + " special waiting pool;";
+ }
+ if (executorAlive(this.threadPool, "thread pool")) {
+ stillAlive = true;
+ culprits = culprits + " thread pool;";
+ }
+
+ if (!stillAlive)
+ return;
+
+ long now = System.currentTimeMillis();
+ if (now >= endTime)
+ break;
+
+ try {
+ Thread.sleep(STOP_PAUSE_TIME);
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ // Desperation, the shutdown thread is being killed. Don't
+ // consult a CancelCriterion.
+ logger.warn(LocalizedMessage.create(
+ LocalizedStrings.DistributionManager_INTERRUPTED_DURING_SHUTDOWN), e);
+ break;
+ }
+ } // for
+
+ logger.warn(LocalizedMessage.create(
+ LocalizedStrings.DistributionManager_DAEMON_THREADS_ARE_SLOW_TO_STOP_CULPRITS_INCLUDE_0, culprits));
+
+ // Kill with no mercy
+ if (this.serialThread != null) {
+ this.serialThread.shutdownNow();
+ }
+ if (this.viewThread != null) {
+ this.viewThread.shutdownNow();
+ }
+ if (this.functionExecutionThread != null) {
+ this.functionExecutionThread.shutdownNow();
+ }
+ if (this.functionExecutionPool != null) {
+ this.functionExecutionPool.shutdownNow();
+ }
+ if (this.partitionedRegionThread != null) {
+ this.partitionedRegionThread.shutdownNow();
+ }
+ if (this.partitionedRegionPool != null) {
+ this.partitionedRegionPool.shutdownNow();
+ }
+ if (this.highPriorityPool != null) {
+ this.highPriorityPool.shutdownNow();
+ }
+ if (this.waitingPool != null) {
+ this.waitingPool.shutdownNow();
+ }
+ if (this.prMetaDataCleanupThreadPool != null) {
+ this.prMetaDataCleanupThreadPool.shutdownNow();
+ }
+ if (this.threadPool != null) {
+ this.threadPool.shutdownNow();
+ }
+
+ Thread th = this.memberEventThread;
+ if (th != null) {
+ clobberThread(th);
+ }
+ }
+
+ private volatile boolean shutdownInProgress = false;
+
+ /** guard for membershipViewIdAcknowledged */
+ private final Object membershipViewIdGuard = new Object();
+
+ /** the latest view ID that has been processed by all membership listeners */
+ private long membershipViewIdAcknowledged;
+
+ public boolean shutdownInProgress() {
+ return this.shutdownInProgress;
+ }
+
+ /**
+ * Stops the pusher, puller and processor threads and closes the
+ * connection to the transport layer. This should only be used from
+ * shutdown() or from the dm initialization code
+ */
+ private void uncleanShutdown(boolean duringStartup)
+ {
+ try {
+ this.closeInProgress = true; // set here also to fix bug 36736
+ removeAllHealthMonitors();
+ shutdownInProgress = true;
+ if (this.channel != null) {
+ this.channel.setShutDown();
+ }
+
+ askThreadsToStop();
+
+ // wait a moment before asking threads to terminate
+ try { waitForThreadsToStop(1000); }
+ catch (InterruptedException ie) {
+ // No need to reset interrupt bit, we're really trying to quit...
+ }
+ forceThreadsToStop();
+
+// // bug36329: desperation measure, send a second interrupt?
+// try { Thread.sleep(1000); }
+// catch (InterruptedException ie) {
+// // No need to reset interrupt bit, we're really trying to quit...
+// }
+// forceThreadsToStop();
+ } // try
+ finally {
+ // ABSOLUTELY ESSENTIAL that we close the distribution channel!
+ try {
+ // For safety, but channel close in a finally AFTER this...
+ if (this.stats != null) {
+ this.stats.close();
+ try { Thread.sleep(100); }
+ catch (InterruptedException ie) {
+ // No need to reset interrupt bit, we're really trying to quit...
+ }
+ }
+ }
+ finally {
+ if (this.channel != null) {
+ logger.info(LocalizedMessage.create(LocalizedStrings.DistributionManager_NOW_CLOSING_DISTRIBUTION_FOR__0, this.myid));
+ this.channel.disconnect(duringStartup);
+// this.channel = null; DO NOT NULL OUT INSTANCE VARIABLES AT SHUTDOWN - bug #42087
+ }
+ }
+ }
+ }
+
+ /**
+ * Returns the distributed system to which this distribution manager
+ * is connected.
+ */
+ public InternalDistributedSystem getSystem() {
+ return this.system;
+ }
+
+ /**
+ * Returns the transport configuration for this distribution manager
+ * @since 5.0
+ */
+ public RemoteTransportConfig getTransport() {
+ return this.transport;
+ }
+
+
+ /**
+ * Adds a MembershipListener to this distribution
+ * manager.
+ */
+ public void addMembershipListener(MembershipListener l) {
+ this.membershipListeners.putIfAbsent(l, Boolean.TRUE);
+ }
+
+ /**
+ * Removes a MembershipListener from this distribution
+ * manager.
+ *
+ * @throws IllegalArgumentException
+ * l was not registered on this distribution
+ * manager
+ */
+ public void removeMembershipListener(MembershipListener l) {
+ this.membershipListeners.remove(l);
+ }
+
+ /**
+ * Adds a MembershipListener to this distribution
+ * manager.
+ * @since 5.7
+ */
+ public void addAllMembershipListener(MembershipListener l) {
+ synchronized (this.allMembershipListenersLock) {
+ Set newAllMembershipListeners =
+ new HashSet(this.allMembershipListeners);
+ newAllMembershipListeners.add(l);
+ this.allMembershipListeners = newAllMembershipListeners;
+ }
+ }
+
+ /**
+ * Removes a MembershipListener listening for all members
+ * from this distribution manager.
+ *
+ * @throws IllegalArgumentException
+ * l was not registered on this distribution
+ * manager
+ * @since 5.7
+ */
+ public void removeAllMembershipListener(MembershipListener l) {
+ synchronized (this.allMembershipListenersLock) {
+ Set newAllMembershipListeners =
+ new HashSet(this.allMembershipListeners);
+ if (!newAllMembershipListeners.remove(l)) {
+ // There seems to be a race condition in which
+ // multiple departure events can be registered
+ // on the same peer. We regard this as benign.
+ // FIXME when membership events become sane again
+// String s = "MembershipListener was never registered";
+// throw new IllegalArgumentException(s);
+ }
+ this.allMembershipListeners = newAllMembershipListeners;
+ }
+ }
+
+ /**
+ * Returns true if this DM or the DistributedSystem owned by
+ * it is closing or is closed.
+ */
+ private boolean isCloseInProgress() {
+ if (closeInProgress) {
+ return true;
+ }
+ InternalDistributedSystem ds = getSystem();
+ if (ds != null && ds.isDisconnecting()) {
+ return true;
+ }
+ return false;
+ }
+
+ private void handleViewInstalledEvent(ViewInstalledEvent ev) {
+ synchronized(this.membershipViewIdGuard) {
+ this.membershipViewIdAcknowledged = ev.getViewId();
+ this.membershipViewIdGuard.notifyAll();
+ }
+ }
+
+ /**
+ * This stalls waiting for the current membership view (as seen by the
+ * membership manager) to be acknowledged by all membership listeners
+ */
+ public void waitForViewInstallation(long id) throws InterruptedException {
+ if (id <= this.membershipViewIdAcknowledged) {
+ return;
+ }
+ synchronized(this.membershipViewIdGuard) {
+ while (this.membershipViewIdAcknowledged < id && this.stopper.cancelInProgress() == null) {
+ if (logger.isDebugEnabled()) {
+ logger.debug("waiting for view {}. Current DM view processed by all listeners is {}", id, this.membershipViewIdAcknowledged);
+ }
+ this.membershipViewIdGuard.wait();
+ }
+ }
+ }
+
+ protected void handleMemberEvent(MemberEvent ev) {
+ ev.handleEvent(this);
+ }
+
+ /**
+ * This thread processes member events as they occur.
+ *
+ * @see com.gemstone.gemfire.distributed.internal.DistributionManager.MemberCrashedEvent
+ * @see com.gemstone.gemfire.distributed.internal.DistributionManager.MemberJoinedEvent
+ * @see com.gemstone.gemfire.distributed.internal.DistributionManager.MemberDepartedEvent
+ *
+ */
+ protected class MemberEventInvoker implements Runnable {
+
+
+ @SuppressWarnings("synthetic-access")
+ public void run() {
+ for (;;) {
+ SystemFailure.checkFailure();
+ // bug 41539 - member events need to be delivered during shutdown
+ // or reply processors may hang waiting for replies from
+ // departed members
+// if (getCancelCriterion().cancelInProgress() != null) {
+// break; // no message, just quit
+// }
+ if (!DistributionManager.this.system.isConnected &&
+ DistributionManager.this.isClosed()) {
+ break;
+ }
+ try {
+ MemberEvent ev = (MemberEvent)DistributionManager.this
+ .membershipEventQueue.take();
+ handleMemberEvent(ev);
+ }
+ catch (InterruptedException e) {
+ if (isCloseInProgress()) {
+ if (logger.isTraceEnabled()) {
+ logger.trace("MemberEventInvoker: InterruptedException during shutdown");
+ }
+ }
+ else {
+ logger.warn(LocalizedMessage.create(LocalizedStrings.DistributionManager_UNEXPECTED_INTERRUPTEDEXCEPTION), e);
+ }
+ break;
+ }
+ catch (DistributedSystemDisconnectedException e) {
+ break;
+ }
+ catch (CancelException e) {
+ if (isCloseInProgress()) {
+ if (logger.isTraceEnabled()) {
+ logger.trace("MemberEventInvoker: cancelled");
+ }
+ }
+ else {
+ logger.warn(LocalizedMessage.create(LocalizedStrings.DistributionManager_UNEXPECTED_CANCELLATION), e);
+ }
+ break;
+ }
+ catch (Exception e) {
+ logger.fatal(LocalizedMessage.create(LocalizedStrings.DistributionManager_UNCAUGHT_EXCEPTION_PROCESSING_MEMBER_EVENT), e);
+ }
+ } // for
+ if (logger.isTraceEnabled()) {
+ logger.trace("MemberEventInvoker on {} stopped", DistributionManager.this);
+ }
+ }
+ }
+
+ private void addMemberEvent(MemberEvent ev) {
+ if (SYNC_EVENTS) {
+ handleMemberEvent(ev);
+ } else {
+ stopper.checkCancelInProgress(null);
+ boolean interrupted = Thread.interrupted();
+ try {
+ this.membershipEventQueue.put(ev);
+ } catch (InterruptedException ex) {
+ interrupted = true;
+ stopper.checkCancelInProgress(ex);
+ handleMemberEvent(ev); // FIXME why???
+ }
+ finally {
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+ }
+
+
+ /**
+ * Stops the threads associated with this distribution manager and
+ * closes the connection to the transport layer.
+ */
+ public void close() {
+ if (!closed) {
+ this.shutdown();
+ logger.info(LocalizedMessage.create(
+ LocalizedStrings.DistributionManager_MARKING_DISTRIBUTIONMANAGER_0_AS_CLOSED, this.myid));
+ MembershipLogger.logShutdown(this.myid);
+ closed = true;
+ synchronized (DistributionManager.class) {
+ openDMs--;
+ }
+ }
+ }
+
+ public void throwIfDistributionStopped() {
+ if (this.shutdownMsgSent) {
+ throw new DistributedSystemDisconnectedException(LocalizedStrings.DistributionManager_MESSAGE_DISTRIBUTION_HAS_TERMINATED.toLocalizedString(), this.getRootCause());
+ }
+ }
+
+ /**
+ * Returns true if this distribution manager has been closed.
+ */
+ public boolean isClosed() {
+ return this.closed;
+ }
+
+ /**
+ * Makes note of a new administration console (admin-only member).
+ */
+ public void addAdminConsole(InternalDistributedMember theId) {
+ logger.info(LocalizedMessage.create(
+ LocalizedStrings.DistributionManager_NEW_ADMINISTRATION_MEMBER_DETECTED_AT_0, theId));
+ synchronized(this.adminConsolesLock) {
+ HashSet tmp = new HashSet(this.adminConsoles);
+ tmp.add(theId);
+ this.adminConsoles = Collections.unmodifiableSet(tmp);
+ }
+ }
+
+ public DMStats getStats() {
+ return this.stats;
+ }
+
+ public DistributionConfig getConfig() {
+ DistributionConfig result = null;
+ InternalDistributedSystem sys = getSystem();
+ if (sys != null) {
+ result = system.getConfig();
+ }
+ return result;
+ }
+
+// /**
+// * Initializes and returns a DistributedSystem to be
+// * sent to new members of the distributed system.
+// *
+// * @since 3.0
+// */
+// protected DistributedState getNewDistributedState() {
+// DistributedState state = new DistributedState();
+// state.setGemFireVersion(GemFireVersion.getGemFireVersion());
+// state.setCacheTime(this.cacheTimeMillis());
+// return state;
+//}
+
+ private static final int STARTUP_TIMEOUT =
+ Integer.getInteger("DistributionManager.STARTUP_TIMEOUT", 15000).intValue();
+
+ public static final boolean DEBUG_NO_ACKNOWLEDGEMENTS = Boolean.getBoolean("DistributionManager.DEBUG_NO_ACKNOWLEDGEMENTS");
+
+ public Set getAllOtherMembers() {
+ Set result = new HashSet(getDistributionManagerIdsIncludingAdmin());
+ result.remove(getDistributionManagerId());
+ return result;
+ }
+
+ @Override // DM method
+ public void retainMembersWithSameOrNewerVersion(Collectionmessage cannot be serialized
+ */
+ protected Set sendOutgoing(DistributionMessage message)
+ throws NotSerializableException {
+ long startTime = DistributionStats.getStatTime();
+
+ Set result = channel.send(message.getRecipients(), message,
+ DistributionManager.this,
+ this.stats);
+ long endTime = 0L;
+ if (DistributionStats.enableClockStats) {
+ endTime = NanoTimer.getTime();
+ }
+ boolean sentToAll = message.forAll();
+
+ if (sentToAll) {
+ stats.incBroadcastMessages(1L);
+ if (DistributionStats.enableClockStats) {
+ stats.incBroadcastMessagesTime(endTime-startTime);
+ }
+ }
+ stats.incSentMessages(1L);
+ if (DistributionStats.enableClockStats) {
+ stats.incSentMessagesTime(endTime-startTime);
+ stats.incDistributeMessageTime(endTime - message.getTimestamp());
+ }
+
+ return result;
+ }
+
+
+
+ /**
+ * @return recipients who did not receive the message
+ * @throws NotSerializableException
+ * If message cannot be serialized
+ */
+ Set sendMessage(DistributionMessage message)
+ throws NotSerializableException {
+ Set result = null;
+ try {
+ // Verify we're not too far into the shutdown
+ stopper.checkCancelInProgress(null);
+
+ // avoid race condition during startup
+ waitUntilReadyToSendMsgs(message);
+
+ result = sendOutgoing(message);
+ } catch (NotSerializableException ex) {
+ throw ex; // serialization error in user data
+ } catch (ToDataException ex) {
+ throw ex; // serialization error in user data
+ }
+ catch (ReenteredConnectException ex) {
+ throw ex; // Recursively tried to get the same connection
+ }
+ catch (CancelException ex) {
+ throw ex; // bug 37194, shutdown conditions
+ }
+ catch (InvalidDeltaException ide) {
+ logger.info(LocalizedMessage.create(LocalizedStrings.DistributionManager_CAUGHT_EXCEPTION_WHILE_SENDING_DELTA), ide.getCause());
+ throw (RuntimeException)ide.getCause();
+ }
+ catch (Exception ex) {
+ DistributionManager.this.exceptionInThreads = true;
+ String receiver = "NULL";
+ if (message != null) {
+ receiver = message.getRecipientsDescription();
+ }
+
+ logger.fatal(LocalizedMessage.create(LocalizedStrings.DistributionManager_WHILE_PUSHING_MESSAGE_0_TO_1, new Object[] {message, receiver}), ex);
+ if (message == null || message.forAll())
+ return null;
+ result = new HashSet();
+ for (int i = 0; i < message.getRecipients().length; i ++)
+ result.add(message.getRecipients()[i]);
+ return result;
+ /* if (ex instanceof com.gemstone.gemfire.GemFireIpcResourceException) {
+ return;
+ }*/
+ }
+ return result;
+ }
+
+
+ /**
+ * Schedule a given message appropriately, depending upon its
+ * executor kind.
+ *
+ * @param message
+ */
+ protected void scheduleIncomingMessage(DistributionMessage message)
+ {
+ /* Potential race condition between starting up and getting other
+ * distribution manager ids -- DM will only be initialized upto
+ * the point at which it called startThreads
+ */
+ waitUntilReadyForMessages();
+ message.schedule(DistributionManager.this);
+ }
+
+ /**
+ * Mutex to control access to {@link #waitingForElderChange}
+ * or {@link #elder}.
+ */
+ protected final Object elderMonitor = new Object();
+
+ /**
+ * Must be read/written while holding {@link #elderMonitor}
+ *
+ * @see #elderChangeWait()
+ */
+ private boolean waitingForElderChange = false;
+
+ /**
+ * @see DM#isAdam()
+ */
+ private boolean adam = false;
+
+ /**
+ * This is the "elder" member of the distributed system, responsible
+ * for certain types of arbitration.
+ *
+ * Must hold {@link #elderMonitor} in order to change this.
+ *
+ * @see #getElderId()
+ */
+ protected volatile InternalDistributedMember elder = null;
+
+ public boolean isAdam() {
+ return this.adam;
+ }
+
+ public InternalDistributedMember getElderId()
+ throws DistributedSystemDisconnectedException {
+// membershipManager.waitForEventProcessing();
+ if (closeInProgress) {
+ throw new DistributedSystemDisconnectedException(LocalizedStrings.DistributionManager_NO_VALID_ELDER_WHEN_SYSTEM_IS_SHUTTING_DOWN.toLocalizedString(), this.getRootCause());
+ }
+ getSystem().getCancelCriterion().checkCancelInProgress(null);
+
+ // Cache a recent value of the elder
+ InternalDistributedMember result = elder;
+ if (result != null && membershipManager.memberExists(result)) {
+ return result;
+ }
+ logger.info(LocalizedMessage.create(LocalizedStrings.DistributionManager_ELDER__0__IS_NOT_CURRENTLY_AN_ACTIVE_MEMBER_SELECTING_NEW_ELDER, elder));
+
+ selectElder(); // ShutdownException can be thrown here
+ logger.info(LocalizedMessage.create(LocalizedStrings.DistributionManager_NEWLY_SELECTED_ELDER_IS_NOW__0_, elder));
+ return elder;
+ }
+
+ public boolean isElder() {
+ return getId().equals(elder);
+ }
+ public boolean isLoner() {
+ return false;
+ }
+
+ private final StoppableReentrantLock elderLock;
+ private ElderState elderState;
+ private volatile boolean elderStateInitialized;
+
+ public ElderState getElderState(boolean force, boolean useTryLock) {
+ if (force) {
+ if (logger.isDebugEnabled()) {
+ if (!this.myid.equals(this.elder)) {
+ logger.debug("Forcing myself, {}, to be the elder.", this.myid);
+ }
+ }
+ changeElder(this.myid);
+ }
+ if (force || this.myid.equals(elder)) {
+ // we are the elder
+ if (this.elderStateInitialized) {
+ return this.elderState;
+ }
+ return getElderStateWithTryLock(useTryLock);
+ } else {
+ // we are not the elder so return null
+ return null;
+ }
+ }
+
+ /**
+ * Usage: GrantorRequestProcessor calls getElderState with useTryLock set
+ * to true if the becomeGrantor Collaboration is already acquired.
+ * null if no admin {@linkplain #getAgent
+ * agent} is associated with this distribution manager
+ *
+ * @since 3.5
+ */
+ public String getDistributionConfigDescription() {
+ if (this.agent == null) {
+ return null;
+
+ } else {
+ return this.agent.getTransport().toString();
+ }
+ }
+
+ /**
+ * A DistributionManager is not intented to be
+ * serialized. This method throws an {@link
+ * UnsupportedOperationException} to prevent a
+ * DistributionManager from being copy shared.
+ */
+ public void writeExternal(ObjectOutput out) throws IOException {
+ throw new UnsupportedOperationException(LocalizedStrings.DistributionManager_DISTRIBUTIONMANAGERS_SHOULD_NOT_BE_COPY_SHARED.toLocalizedString());
+ }
+
+ /**
+ * A DistributionManager is not intented to be
+ * serialized. This method throws an {@link
+ * UnsupportedOperationException} to prevent a
+ * DistributionManager from being copy shared.
+ */
+ public void readExternal(ObjectInput out)
+ throws IOException, ClassNotFoundException {
+ throw new UnsupportedOperationException(LocalizedStrings.DistributionManager_DISTRIBUTIONMANAGERS_SHOULD_NOT_BE_COPY_SHARED.toLocalizedString());
+ }
+
+ /* -----------------------------Health Monitor------------------------------ */
+ private final ConcurrentMap hmMap = new ConcurrentHashMap();
+
+ /**
+ * Returns the health monitor for this distribution manager and owner.
+ * @param owner the agent that owns the returned monitor
+ * @return the health monitor created by the owner; null
+ * if the owner has now created a monitor.
+ * @since 3.5
+ */
+ public HealthMonitor getHealthMonitor(InternalDistributedMember owner) {
+ return (HealthMonitor)this.hmMap.get(owner);
+ }
+ /**
+ * Returns the health monitor for this distribution manager.
+ *
+ * @param owner the agent that owns the created monitor
+ * @param cfg the configuration to use when creating the monitor
+ * @since 3.5
+ */
+ public void createHealthMonitor(InternalDistributedMember owner,
+ GemFireHealthConfig cfg) {
+ if (closeInProgress) {
+ return;
+ }
+ {
+ final HealthMonitor hm = getHealthMonitor(owner);
+ if (hm != null) {
+ hm.stop();
+ this.hmMap.remove(owner);
+ }
+ }
+ {
+ HealthMonitorImpl newHm = new HealthMonitorImpl(owner, cfg, this);
+ newHm.start();
+ this.hmMap.put(owner, newHm);
+ }
+ }
+ /**
+ * Remove a monitor that was previously created.
+ * @param owner the agent that owns the monitor to remove
+ */
+ public void removeHealthMonitor(InternalDistributedMember owner, int theId) {
+ final HealthMonitor hm = getHealthMonitor(owner);
+ if (hm != null && hm.getId() == theId) {
+ hm.stop();
+ this.hmMap.remove(owner);
+ }
+ }
+ public void removeAllHealthMonitors() {
+ Iterator it = this.hmMap.values().iterator();
+ while (it.hasNext()) {
+ HealthMonitor hm = (HealthMonitor)it.next();
+ hm.stop();
+ it.remove();
+ }
+ }
+
+ // For feature request #32887
+ public Set getAdminMemberSet() {
+ return this.adminConsoles;
+ }
+
+ /** Returns count of members filling the specified role */
+ public int getRoleCount(Role role) {
+ int count = 0;
+ Set mbrs = getDistributionManagerIds();
+ for (Iterator mbrIter = mbrs.iterator(); mbrIter.hasNext();) {
+ Set roles = ((InternalDistributedMember) mbrIter.next()).getRoles();
+ for (Iterator rolesIter = roles.iterator(); rolesIter.hasNext();) {
+ Role mbrRole = (Role) rolesIter.next();
+ if (mbrRole.equals(role)) {
+ count++;
+ break;
+ }
+ }
+ }
+ return count;
+ }
+
+ /** Returns true if at least one member is filling the specified role */
+ public boolean isRolePresent(Role role) {
+ Set mbrs = getDistributionManagerIds();
+ for (Iterator mbrIter = mbrs.iterator(); mbrIter.hasNext();) {
+ Set roles = ((InternalDistributedMember) mbrIter.next()).getRoles();
+ for (Iterator rolesIter = roles.iterator(); rolesIter.hasNext();) {
+ Role mbrRole = (Role) rolesIter.next();
+ if (mbrRole.equals(role)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /** Returns a set of all roles currently in the distributed system. */
+ public Set getAllRoles() {
+ Set allRoles = new HashSet();
+ Set mbrs = getDistributionManagerIds();
+ for (Iterator mbrIter = mbrs.iterator(); mbrIter.hasNext();) {
+ Set roles = ((InternalDistributedMember) mbrIter.next()).getRoles();
+ for (Iterator rolesIter = roles.iterator(); rolesIter.hasNext();) {
+ Role mbrRole = (Role) rolesIter.next();
+ allRoles.add(mbrRole);
+ }
+ }
+ return allRoles;
+ }
+
+ /** Returns the membership manager for this distributed system.
+ The membership manager owns the membership set and handles
+ all communications. The manager should NOT be used to
+ bypass DistributionManager to send or receive messages.BoundedLinkedHashMap is a bounded
+ * LinkedHashMap. The bound is the maximum
+ * number of entries the BoundedLinkedHashMap
+ * can contain.
+ */
+ static class BoundedLinkedHashMap extends LinkedHashMap
+ {
+ private static final long serialVersionUID = -3419897166186852692L;
+
+ /**
+ * The maximum number of entries allowed in this
+ * BoundedLinkedHashMap
+ */
+ protected int _maximumNumberOfEntries;
+
+ /**
+ * Constructor.
+ *
+ * @param maximumNumberOfEntries The maximum number of allowed entries
+ */
+ public BoundedLinkedHashMap(int maximumNumberOfEntries) {
+ super();
+ this._maximumNumberOfEntries = maximumNumberOfEntries;
+ }
+
+ @Override
+ protected boolean removeEldestEntry(Map.Entry entry) {
+ return size() > this._maximumNumberOfEntries;
+ }
+ }
+
+}
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/MigrationServer.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/MigrationServer.java
new file mode 100755
index 000000000000..3d6d6bef0750
--- /dev/null
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/MigrationServer.java
@@ -0,0 +1,565 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gemstone.gemfire.internal;
+
+import com.gemstone.gemfire.admin.internal.InetAddressUtil;
+import com.gemstone.gemfire.cache.Cache;
+import com.gemstone.gemfire.cache.CacheFactory;
+import com.gemstone.gemfire.cache.Region;
+import com.gemstone.gemfire.distributed.DistributedSystem;
+import com.gemstone.gemfire.distributed.internal.DistributionConfig;
+import com.gemstone.gemfire.distributed.internal.InternalDistributedSystem;
+import com.gemstone.gemfire.internal.cache.GemFireCacheImpl;
+import com.gemstone.gemfire.internal.i18n.LocalizedStrings;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.NetworkInterface;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.SocketAddress;
+import java.net.SocketException;
+import java.util.Enumeration;
+import java.util.Iterator;
+import java.util.Properties;
+
+/**
+ * MigrationServer creates a cache using a supplied cache.xml and then
+ * opens a server socket that a MigrationClient connects to and requests
+ * the data from a Region. MigrationServer sends the data to
+ * the MigrationClient using normal java serialization in
+ * order to allow migration from incompatible versions of DataSerializer.
+ * Keys and values stored in the cache must serialize and deserialize correctly.
+ *
+ * cache-xml-file-name (required)
+ * listen port (defaults to 10553)
+ * bind address (defaults to listing on all interfaces)
+ *
+ * java -cp $MYCLASSES:migration.jar:$GEMFIRE/lib/gemfire-core-dependencies.jar \
+ * com.gemstone.gemfire.internal.MigrationServer cacheDescription.xml
+ *
+ * <!DOCTYPE cache PUBLIC
+ "-//GemStone Systems, Inc.//GemFire Declarative Caching 5.7//EN"
+ "http://www.gemstone.com/dtd/cache5_7.dtd">
+<cache is-server="false">
+ <region name="root">
+ <region-attributes scope="distributed-no-ack">
+ </region-attributes>
+
+ <region name="Test">
+ <region-attributes data-policy="persistent-replicate">
+
+ <disk-write-attributes>
+ <synchronous-writes/>
+ </disk-write-attributes>
+
+ <disk-dirs>
+ <disk-dir>diskfiles</disk-dir>
+ </disk-dirs>
+
+ <eviction-attributes>
+ <lru-memory-size maximum="100" action="overflow-to-disk"/>
+ </eviction-attributes>
+
+ </region-attributes>
+ </region> <!-- Test region -->
+ </region> <!-- root region -->
+</cache>
+
+ *
DiskEntry has a unique id that is
+ * used by the {@link DiskRegion} to identify the key/value pair.
+ * Before the disk entry is written to disk, the value of the
+ * id is {@link DiskRegion#INVALID_ID invalid}. Once the
+ * object has been written to disk, the id is a positive
+ * number. If the value is {@linkplain Helper#update updated}, then the
+ * id is negated to signify that the value on disk is
+ * dirty.
+ *
+ * @see DiskRegion
+ *
+ *
+ * @since 3.2
+ */
+public interface DiskEntry extends RegionEntry {
+ /**
+ * Sets the value with a {@link RegionEntryContext}.
+ * @param context the value's context.
+ * @param value an entry value.
+ */
+ public void setValueWithContext(RegionEntryContext context,Object value);
+
+ /**
+ * In some cases we need to do something just before we drop the value
+ * from a DiskEntry that is being moved (i.e. overflowed) to disk.
+ * @param context
+ */
+ public void handleValueOverflow(RegionEntryContext context);
+
+ /**
+ * In some cases we need to do something just after we unset the value
+ * from a DiskEntry that has been moved (i.e. overflowed) to disk.
+ * @param context
+ */
+ public void afterValueOverflow(RegionEntryContext context);
+
+ /**
+ * Returns true if the DiskEntry value is equal to {@link Token#DESTROYED}, {@link Token#REMOVED_PHASE1}, or {@link Token#REMOVED_PHASE2}.
+ */
+ public boolean isRemovedFromDisk();
+
+ /**
+ * Returns the id of this DiskEntry
+ */
+ public DiskId getDiskId();
+
+ public void _removePhase1();
+
+ public int updateAsyncEntrySize(EnableLRU capacityController);
+
+ public DiskEntry getPrev();
+ public DiskEntry getNext();
+ public void setPrev(DiskEntry v);
+ public void setNext(DiskEntry v);
+
+ /**
+ * Used as the entry value if it was invalidated.
+ */
+ public static final byte[] INVALID_BYTES = new byte[0];
+ /**
+ * Used as the entry value if it was locally invalidated.
+ */
+ public static final byte[] LOCAL_INVALID_BYTES = new byte[0];
+ /**
+ * Used as the entry value if it was tombstone.
+ */
+ public static final byte[] TOMBSTONE_BYTES = new byte[0];
+
+ /////////////////////// Inner Classes //////////////////////
+
+ /**
+ * A Helper class for performing functions common to all
+ * DiskEntrys.
+ */
+ public static class Helper {
+ private static final Logger logger = LogService.getLogger();
+
+ /**
+ * Testing purpose only
+ * Get the value of an entry that is on disk without faulting
+ * it in and without looking in the io buffer.
+ * @since 3.2.1
+ */
+ static Object getValueOnDisk(DiskEntry entry, DiskRegion dr) {
+ DiskId id = entry.getDiskId();
+ if (id == null) {
+ return null;
+ }
+ dr.acquireReadLock();
+ try {
+ synchronized (id) {
+ if (id == null
+ || (dr.isBackup() && id.getKeyId() == DiskRegion.INVALID_ID)
+ || (!entry.isValueNull() && id.needsToBeWritten() && !EntryBits.isRecoveredFromDisk(id.getUserBits()))/*fix for bug 41942*/) {
+ return null;
+ }
+
+ return dr.getNoBuffer(id);
+ }
+ } finally {
+ dr.releaseReadLock();
+ }
+ }
+
+ /**
+ * Get the serialized value directly from disk. Returned object may be
+ * a {@link CachedDeserializable}. Goes straight to disk without faulting
+ * into memory. Only looks at the disk storage, not at heap storage.
+ * @param entry the entry used to identify the value to fetch
+ * @param dr the persistent storage from which to fetch the value
+ * @return either null, byte array, or CacheDeserializable
+ * @since gemfire57_hotfix
+ */
+ public static Object getSerializedValueOnDisk(
+ DiskEntry entry, DiskRegion dr) {
+ DiskId did = entry.getDiskId();
+ if (did == null) {
+ return null;
+ }
+ dr.acquireReadLock();
+ try {
+ synchronized (did) {
+ if (did == null
+ || (dr.isBackup() && did.getKeyId() == DiskRegion.INVALID_ID)) {
+ return null;
+ } else if (!entry.isValueNull() && did.needsToBeWritten() && !EntryBits.isRecoveredFromDisk(did.getUserBits())/*fix for bug 41942*/) {
+ return null;
+ }
+ return dr.getSerializedData(did);
+ }
+ } finally {
+ dr.releaseReadLock();
+ }
+ }
+
+
+ /**
+ * Get the value of an entry that is on disk without
+ * faulting it in . It checks for the presence in the buffer also.
+ * This method is used for concurrent map operations, SQLFabric and CQ processing
+ *
+ * @throws DiskAccessException
+ * @since 5.1
+ */
+ static Object getValueOnDiskOrBuffer(DiskEntry entry, DiskRegion dr, RegionEntryContext context) {
+ @Released Object v = getOffHeapValueOnDiskOrBuffer(entry, dr, context);
+ if (v instanceof CachedDeserializable) {
+ if (v instanceof Chunk) {
+ @Released Chunk ohv = (Chunk) v;
+ try {
+ v = ohv.getDeserializedValue(null, null);
+ if (v == ohv) {
+ throw new IllegalStateException("sqlf tried to use getValueOnDiskOrBuffer");
+ }
+ } finally {
+ ohv.release(); // OFFHEAP the offheap ref is decremented here
+ }
+ } else {
+ v = ((CachedDeserializable)v).getDeserializedValue(null, null);
+ }
+ }
+ return v;
+ }
+
+ @Retained
+ static Object getOffHeapValueOnDiskOrBuffer(DiskEntry entry, DiskRegion dr, RegionEntryContext context) {
+ DiskId did = entry.getDiskId();
+ Object syncObj = did;
+ if (syncObj == null) {
+ syncObj = entry;
+ }
+ if (syncObj == did) {
+ dr.acquireReadLock();
+ }
+ try {
+ synchronized (syncObj) {
+ if (did != null && did.isPendingAsync()) {
+ @Retained Object v = entry._getValueRetain(context, true); // TODO:KIRK:OK Rusty had Object v = entry.getValueWithContext(context);
+ if (Token.isRemovedFromDisk(v)) {
+ v = null;
+ }
+ return v;
+ }
+ if (did == null
+ || ( dr.isBackup() && did.getKeyId() == DiskRegion.INVALID_ID)
+ || (!entry.isValueNull() && did.needsToBeWritten() && !EntryBits.isRecoveredFromDisk(did.getUserBits()))/*fix for bug 41942*/) {
+ return null;
+ }
+
+ return dr.getSerializedData(did);
+ }
+ } finally {
+ if (syncObj == did) {
+ dr.releaseReadLock();
+ }
+ }
+ }
+
+ /**
+ * Returns false if the entry is INVALID (or LOCAL_INVALID). Determines this
+ * without faulting in the value from disk.
+ *
+ * @since 3.2.1
+ */
+ /* TODO prpersist - Do we need this method? It was added by the sqlf merge
+ static boolean isValid(DiskEntry entry, DiskRegion dr) {
+ synchronized (entry) {
+ if (entry.isRecovered()) {
+ // We have a recovered entry whose value is still on disk.
+ // So take a peek at it without faulting it in.
+ //long id = entry.getDiskId().getKeyId();
+ //entry.getDiskId().setKeyId(-id);
+ byte bits = dr.getBits(entry.getDiskId());
+ //TODO Asif:Check if resetting is needed
+ return !EntryBits.isInvalid(bits) && !EntryBits.isLocalInvalid(bits);
+ }
+ }
+ }*/
+
+ static boolean isOverflowedToDisk(DiskEntry de, DiskRegion dr, DistributedRegion.DiskPosition dp,RegionEntryContext context) {
+ Object v = null;
+ DiskId did;
+ synchronized (de) {
+ did = de.getDiskId();
+ }
+ Object syncObj = did;
+ if (syncObj == null) {
+ syncObj = de;
+ }
+ if (syncObj == did) {
+ dr.acquireReadLock();
+ }
+ try {
+ synchronized (syncObj) {
+ if (de.isValueNull()) {
+ if (did == null) {
+ synchronized (de) {
+ did = de.getDiskId();
+ }
+ assert did != null;
+ return isOverflowedToDisk(de, dr, dp, context);
+ } else {
+ dp.setPosition(did.getOplogId(), did.getOffsetInOplog());
+ return true;
+ }
+ } else {
+ return false;
+ }
+ }
+ } finally {
+ if (syncObj == did) {
+ dr.releaseReadLock();
+ }
+ }
+ }
+
+ /**
+ * Get the value of an entry that is on disk without faulting
+ * it in.
+ * @since 3.2.1
+ */
+ static boolean fillInValue(DiskEntry de, InitialImageOperation.Entry entry,
+ DiskRegion dr, DM mgr, ByteArrayDataInput in, RegionEntryContext context) {
+ @Retained @Released Object v = null;
+ DiskId did;
+ synchronized (de) {
+ did = de.getDiskId();
+ }
+ Object syncObj = did;
+ if (syncObj == null) {
+ syncObj = de;
+ }
+ if (syncObj == did) {
+ dr.acquireReadLock();
+ }
+ try {
+ synchronized (syncObj) {
+ entry.setLastModified(mgr, de.getLastModified());
+
+ ReferenceCountHelper.setReferenceCountOwner(entry);
+ v = de._getValueRetain(context, true); // OFFHEAP copied to heap entry; todo allow entry to refer to offheap since it will be copied to network.
+ ReferenceCountHelper.setReferenceCountOwner(null);
+ if (v == null) {
+ if (did == null) {
+ // fix for bug 41449
+ synchronized (de) {
+ did = de.getDiskId();
+ }
+ assert did != null;
+ // do recursive call to get readLock on did
+ return fillInValue(de, entry, dr, mgr, in, context);
+ }
+ if (logger.isDebugEnabled()) {
+ logger.debug("DiskEntry.Helper.fillInValue, key={}; getting value from disk, disk id={}", entry.key, did);
+ }
+ BytesAndBits bb = null;
+ try {
+ bb = dr.getBytesAndBits(did, false);
+ }catch(DiskAccessException dae){
+ return false;
+ }
+ if (EntryBits.isInvalid(bb.getBits())) {
+ entry.setInvalid();
+ }
+ else if (EntryBits.isLocalInvalid(bb.getBits())) {
+ entry.setLocalInvalid();
+ }
+ else if (EntryBits.isTombstone(bb.getBits())) {
+ entry.setTombstone();
+ }
+ else {
+ entry.value = bb.getBytes();
+ entry.setSerialized(EntryBits.isSerialized(bb.getBits()));
+ }
+ return true;
+ }
+ }
+ } finally {
+ if (syncObj == did) {
+ dr.releaseReadLock();
+ }
+ }
+ final boolean isEagerDeserialize = entry.isEagerDeserialize();
+ if (isEagerDeserialize) {
+ entry.clearEagerDeserialize();
+ }
+ if (Token.isRemovedFromDisk(v)) {
+ // fix for bug 31757
+ return false;
+ } else if (v instanceof CachedDeserializable) {
+ try {
+ if (v instanceof StoredObject && !((StoredObject) v).isSerialized()) {
+ entry.setSerialized(false);
+ entry.value = ((StoredObject) v).getDeserializedForReading();
+
+ //For SQLFire we prefer eager deserialized
+// if(v instanceof ByteSource) {
+// entry.setEagerDeserialize();
+// }
+ } else {
+ // don't serialize here if it is not already serialized
+
+ Object tmp = ((CachedDeserializable)v).getValue();
+ //For SQLFire we prefer eager deserialized
+// if(v instanceof ByteSource) {
+// entry.setEagerDeserialize();
+// }
+ if (tmp instanceof byte[]) {
+ byte[] bb = (byte[])tmp;
+ entry.value = bb;
+ entry.setSerialized(true);
+ }
+ else if (isEagerDeserialize && tmp instanceof byte[][]) {
+ // optimize for byte[][] since it will need to be eagerly deserialized
+ // for SQLFabric
+ entry.value = tmp;
+ entry.setEagerDeserialize();
+ entry.setSerialized(true);
+ }
+ else {
+ try {
+ HeapDataOutputStream hdos = new HeapDataOutputStream(Version.CURRENT);
+ BlobHelper.serializeTo(tmp, hdos);
+ hdos.trim();
+ entry.value = hdos;
+ entry.setSerialized(true);
+ } catch (IOException e) {
+ RuntimeException e2 = new IllegalArgumentException(LocalizedStrings.DiskEntry_AN_IOEXCEPTION_WAS_THROWN_WHILE_SERIALIZING.toLocalizedString());
+ e2.initCause(e);
+ throw e2;
+ }
+ }
+ }
+ } finally {
+ // If v == entry.value then v is assumed to be an OffHeapByteSource
+ // and release() will be called on v after the bytes have been read from
+ // off-heap.
+ if (v != entry.value) {
+ OffHeapHelper.releaseWithNoTracking(v);
+ }
+ }
+ }
+ else if (v instanceof byte[]) {
+ entry.value = v;
+ entry.setSerialized(false);
+ }
+ else if (isEagerDeserialize && v instanceof byte[][]) {
+ // optimize for byte[][] since it will need to be eagerly deserialized
+ // for SQLFabric
+ entry.value = v;
+ entry.setEagerDeserialize();
+ }
+ else if (v == Token.INVALID) {
+ entry.setInvalid();
+ }
+ else if (v == Token.LOCAL_INVALID) {
+ // fix for bug 31107
+ entry.setLocalInvalid();
+ } else if (v == Token.TOMBSTONE) {
+ entry.setTombstone();
+ }
+ else {
+ Object preparedValue = v;
+ if (preparedValue != null) {
+ preparedValue = AbstractRegionEntry.prepareValueForGII(preparedValue);
+ if (preparedValue == null) {
+ return false;
+ }
+ }
+ if (CachedDeserializableFactory.preferObject()) {
+ entry.value = preparedValue;
+ entry.setEagerDeserialize();
+ }
+ else {
+ try {
+ HeapDataOutputStream hdos = new HeapDataOutputStream(Version.CURRENT);
+ BlobHelper.serializeTo(preparedValue, hdos);
+ hdos.trim();
+ entry.value = hdos;
+ entry.setSerialized(true);
+ } catch (IOException e) {
+ RuntimeException e2 = new IllegalArgumentException(LocalizedStrings.DiskEntry_AN_IOEXCEPTION_WAS_THROWN_WHILE_SERIALIZING.toLocalizedString());
+ e2.initCause(e);
+ throw e2;
+ }
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Used to initialize a new disk entry
+ */
+ static void initialize(DiskEntry entry, DiskRecoveryStore r, Object newValue) {
+ DiskRegionView drv = null;
+ if (r instanceof LocalRegion) {
+ drv = ((LocalRegion)r).getDiskRegion();
+ } else if (r instanceof DiskRegionView) {
+ drv = (DiskRegionView)r;
+ }
+ if (drv == null) {
+ throw new IllegalArgumentException(LocalizedStrings.DiskEntry_DISK_REGION_IS_NULL.toLocalizedString());
+ }
+
+ if (newValue == null || Token.isRemovedFromDisk(newValue)) {
+ // it is not in vm and it is not on disk
+ DiskId did = entry.getDiskId();
+ if (did != null) {
+ did.setKeyId(DiskRegion.INVALID_ID);
+ }
+ }
+ else if (newValue instanceof RecoveredEntry) {
+ // Set the id directly, the value will also be set if RECOVER_VALUES
+ RecoveredEntry re = (RecoveredEntry)newValue;
+ DiskId did = entry.getDiskId();
+ did.setOplogId(re.getOplogId());
+ did.setOffsetInOplog(re.getOffsetInOplog());
+ did.setKeyId(re.getRecoveredKeyId());
+ did.setUserBits(re.getUserBits());
+ did.setValueLength(re.getValueLength());
+ if (re.getRecoveredKeyId() < 0) {
+ drv.incNumOverflowOnDisk(1L);
+ drv.incNumOverflowBytesOnDisk(did.getValueLength());
+ incrementBucketStats(r, 0/*InVM*/, 1/*OnDisk*/, did.getValueLength());
+ }
+ else {
+ entry.setValueWithContext(drv, entry.prepareValueForCache((RegionEntryContext) r,
+ re.getValue(), false));
+ drv.incNumEntriesInVM(1L);
+ incrementBucketStats(r, 1/*InVM*/, 0/*OnDisk*/, 0);
+ }
+ }
+ else {
+ DiskId did = entry.getDiskId();
+ if (did != null) {
+ did.setKeyId(DiskRegion.INVALID_ID);
+ }
+ drv.incNumEntriesInVM(1L);
+ incrementBucketStats(r, 1/*InVM*/, 0/*OnDisk*/, 0);
+ }
+ }
+
+ private static final ValueWrapper INVALID_VW = new ByteArrayValueWrapper(true, INVALID_BYTES);
+ private static final ValueWrapper LOCAL_INVALID_VW = new ByteArrayValueWrapper(true, LOCAL_INVALID_BYTES);
+ private static final ValueWrapper TOMBSTONE_VW = new ByteArrayValueWrapper(true, TOMBSTONE_BYTES);
+
+ public static interface ValueWrapper {
+ public boolean isSerialized();
+ public int getLength();
+ public byte getUserBits();
+ public void sendTo(ByteBuffer bb, Flushable flushable) throws IOException;
+ public String getBytesAsString();
+ }
+ public static interface Flushable {
+ public void flush() throws IOException;
+
+ public void flush(ByteBuffer bb, ByteBuffer chunkbb) throws IOException;
+ }
+ public static class ByteArrayValueWrapper implements ValueWrapper {
+ public final boolean isSerializedObject;
+ public final byte[] bytes;
+
+ public ByteArrayValueWrapper(boolean isSerializedObject, byte[] bytes) {
+ this.isSerializedObject = isSerializedObject;
+ this.bytes = bytes;
+ }
+
+ @Override
+ public boolean isSerialized() {
+ return this.isSerializedObject;
+ }
+
+ @Override
+ public int getLength() {
+ return (this.bytes != null) ? this.bytes.length : 0;
+ }
+
+ private boolean isInvalidToken() {
+ return this == INVALID_VW;
+ }
+
+ private boolean isLocalInvalidToken() {
+ return this == LOCAL_INVALID_VW;
+ }
+
+ private boolean isTombstoneToken() {
+ return this == TOMBSTONE_VW;
+ }
+
+ @Override
+ public byte getUserBits() {
+ byte userBits = 0x0;
+ if (isSerialized()) {
+ if (isTombstoneToken()) {
+ userBits = EntryBits.setTombstone(userBits, true);
+ } else if (isInvalidToken()) {
+ userBits = EntryBits.setInvalid(userBits, true);
+ } else if (isLocalInvalidToken()) {
+ userBits = EntryBits.setLocalInvalid(userBits, true);
+ } else {
+ if (this.bytes == null) {
+ throw new IllegalStateException("userBits==1 and value is null");
+ } else if (this.bytes.length == 0) {
+ throw new IllegalStateException("userBits==1 and value is zero length");
+ }
+ userBits = EntryBits.setSerialized(userBits, true);
+ }
+ }
+ return userBits;
+ }
+
+ @Override
+ public void sendTo(ByteBuffer bb, Flushable flushable) throws IOException {
+ int offset = 0;
+ final int maxOffset = getLength();
+ while (offset < maxOffset) {
+ int bytesThisTime = maxOffset - offset;
+ boolean needsFlush = false;
+ if (bytesThisTime > bb.remaining()) {
+ needsFlush = true;
+ bytesThisTime = bb.remaining();
+ }
+ bb.put(this.bytes, offset, bytesThisTime);
+ offset += bytesThisTime;
+ if (needsFlush) {
+ flushable.flush();
+ }
+ }
+ }
+
+ @Override
+ public String getBytesAsString() {
+ if (this.bytes == null) {
+ return "null";
+ }
+ StringBuffer sb = new StringBuffer();
+ int len = getLength();
+ for (int i = 0; i < len; i++) {
+ sb.append(this.bytes[i]).append(", ");
+ }
+ return sb.toString();
+ }
+ }
+
+ /**
+ * This class is a bit of a hack used by the compactor.
+ * For the compactor always copies to a byte[] so
+ * this class is just a simple wrapper.
+ * It is possible that the length of the byte array is greater
+ * than the actual length of the wrapped data.
+ * At the time we create this we are all done with isSerialized
+ * and userBits so those methods are not supported.
+ */
+ public static class CompactorValueWrapper extends ByteArrayValueWrapper {
+ private final int length;
+
+ public CompactorValueWrapper(byte[] bytes, int length) {
+ super(false, bytes);
+ this.length = length;
+ }
+
+ @Override
+ public boolean isSerialized() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int getLength() {
+ return this.length;
+ }
+
+ @Override
+ public byte getUserBits() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ /**
+ * Note that the Chunk this ValueWrapper is created with
+ * is unretained so it must be used before the owner of
+ * the chunk releases it.
+ * Since the RegionEntry that has the value we are writing to
+ * disk has it retained we are ok as long as this ValueWrapper's
+ * life ends before the RegionEntry sync is released.
+ * Note that this class is only used with uncompressed chunks.
+ */
+ public static class ChunkValueWrapper implements ValueWrapper {
+ private final @Unretained Chunk chunk;
+ public ChunkValueWrapper(Chunk c) {
+ assert !c.isCompressed();
+ this.chunk = c;
+ }
+ @Override
+ public boolean isSerialized() {
+ return this.chunk.isSerialized();
+ }
+ @Override
+ public int getLength() {
+ return this.chunk.getDataSize();
+ }
+ @Override
+ public byte getUserBits() {
+ byte userBits = 0x0;
+ if (isSerialized()) {
+ userBits = EntryBits.setSerialized(userBits, true);
+ }
+ return userBits;
+ }
+ @Override
+ public void sendTo(ByteBuffer bb, Flushable flushable) throws IOException {
+ final int maxOffset = getLength();
+ if (maxOffset == 0) {
+ return;
+ }
+ if (maxOffset > bb.capacity()) {
+ ByteBuffer chunkbb = this.chunk.createDirectByteBuffer();
+ if (chunkbb != null) {
+ flushable.flush(bb, chunkbb);
+ return;
+ }
+ }
+ final long bbAddress = Chunk.getDirectByteBufferAddress(bb);
+ if (bbAddress != 0L) {
+ int bytesRemaining = maxOffset;
+ int availableSpace = bb.remaining();
+ long addrToWrite = bbAddress + bb.position();
+ long addrToRead = this.chunk.getAddressForReading(0, maxOffset);
+ if (bytesRemaining > availableSpace) {
+ do {
+ UnsafeMemoryChunk.copyMemory(addrToRead, addrToWrite, availableSpace);
+ bb.position(bb.position()+availableSpace);
+ addrToRead += availableSpace;
+ bytesRemaining -= availableSpace;
+ flushable.flush();
+ addrToWrite = bbAddress + bb.position();
+ availableSpace = bb.remaining();
+ } while (bytesRemaining > availableSpace);
+ }
+ UnsafeMemoryChunk.copyMemory(addrToRead, addrToWrite, bytesRemaining);
+ bb.position(bb.position()+bytesRemaining);
+ } else {
+ long addr = this.chunk.getAddressForReading(0, maxOffset);
+ final long endAddr = addr + maxOffset;
+ while (addr != endAddr) {
+ bb.put(UnsafeMemoryChunk.readAbsoluteByte(addr));
+ addr++;
+ if (!bb.hasRemaining()) {
+ flushable.flush();
+ }
+ }
+ }
+ }
+ @Override
+ public String getBytesAsString() {
+ return this.chunk.getStringForm();
+ }
+ }
+
+ public static ValueWrapper createValueWrapper(Object value, EntryEventImpl event) {
+ if (value == Token.INVALID) {
+ // even though it is not serialized we say it is because
+ // bytes will never be an empty array when it is serialized
+ // so that gives us a way to specify the invalid value
+ // given a byte array and a boolean flag.
+ return INVALID_VW;
+ }
+ else if (value == Token.LOCAL_INVALID) {
+ // even though it is not serialized we say it is because
+ // bytes will never be an empty array when it is serialized
+ // so that gives us a way to specify the local-invalid value
+ // given a byte array and a boolean flag.
+ return LOCAL_INVALID_VW;
+ }
+ else if (value == Token.TOMBSTONE) {
+ return TOMBSTONE_VW;
+ }
+ else {
+ boolean isSerializedObject = true;
+ byte[] bytes;
+ if (value instanceof CachedDeserializable) {
+ CachedDeserializable proxy = (CachedDeserializable)value;
+ if (proxy instanceof Chunk) {
+ return new ChunkValueWrapper((Chunk) proxy);
+ }
+ if (proxy instanceof StoredObject) {
+ StoredObject ohproxy = (StoredObject) proxy;
+ isSerializedObject = ohproxy.isSerialized();
+ if (isSerializedObject) {
+ bytes = ohproxy.getSerializedValue();
+ } else {
+ bytes = (byte[]) ohproxy.getDeserializedForReading();
+ }
+ } else {
+ bytes = proxy.getSerializedValue();
+ }
+ if (event != null && isSerializedObject) {
+ event.setCachedSerializedNewValue(bytes);
+ }
+ }
+ else if (value instanceof byte[]) {
+ isSerializedObject = false;
+ bytes = (byte[])value;
+ }
+ else {
+ Assert.assertTrue(!Token.isRemovedFromDisk(value));
+ if (event != null && event.getCachedSerializedNewValue() != null) {
+ bytes = event.getCachedSerializedNewValue();
+ } else {
+ bytes = EntryEventImpl.serialize(value);
+ if (bytes.length == 0) {
+ throw new IllegalStateException("serializing <" + value + "> produced empty byte array");
+ }
+ if (event != null) {
+ event.setCachedSerializedNewValue(bytes);
+ }
+ }
+ }
+ return new ByteArrayValueWrapper(isSerializedObject, bytes);
+ }
+ }
+ public static ValueWrapper createValueWrapperFromEntry(DiskEntry entry, LocalRegion region, EntryEventImpl event) {
+ if (event != null) {
+ // For off-heap it should be faster to pass a reference to the
+ // StoredObject instead of using the cached byte[] (unless it is also compressed).
+ // Since NIO is used if the chunk of memory is large we can write it
+ // to the file with using the off-heap memory with no extra copying.
+ // So we give preference to getRawNewValue over getCachedSerializedNewValue
+ Object rawValue = null;
+ if (!event.hasDelta()) {
+ // We don't do this for the delta case because getRawNewValue returns delta
+ // and we want to write the entire new value to disk.
+ rawValue = event.getRawNewValue();
+ if (rawValue instanceof Chunk) {
+ return new ChunkValueWrapper((Chunk) rawValue);
+ }
+ }
+ if (event.getCachedSerializedNewValue() != null) {
+ return new ByteArrayValueWrapper(true, event.getCachedSerializedNewValue());
+ }
+ if (rawValue != null) {
+ return createValueWrapper(rawValue, event);
+ }
+ }
+ // TODO OFFHEAP: No need to retain since we hold the sync on entry but we need a flavor of _getValue that will decompress
+ @Retained Object value = entry._getValueRetain(region, true);
+ try {
+ return createValueWrapper(value, event);
+ } finally {
+ OffHeapHelper.release(value);
+ }
+ }
+
+ private static void writeToDisk(DiskEntry entry, LocalRegion region, boolean async) throws RegionClearedException {
+ writeToDisk(entry, region, async, null);
+ }
+
+ /**
+ * Writes the key/value object stored in the given entry to disk
+ * @throws RegionClearedException
+ *
+ * @see DiskRegion#put
+ */
+ private static void writeToDisk(DiskEntry entry, LocalRegion region, boolean async, EntryEventImpl event) throws RegionClearedException {
+ writeBytesToDisk(entry, region, async, createValueWrapperFromEntry(entry, region, event));
+ }
+
+ private static void writeBytesToDisk(DiskEntry entry, LocalRegion region, boolean async, ValueWrapper vw) throws RegionClearedException {
+ // @todo does the following unmark need to be called when an async
+ // write is scheduled or is it ok for doAsyncFlush to do it?
+ entry.getDiskId().unmarkForWriting();
+ region.getDiskRegion().put(entry, region, vw, async);
+ }
+
+ public static void update(DiskEntry entry, LocalRegion region, Object newValue) throws RegionClearedException {
+ update(entry, region, newValue, null);
+ }
+ /**
+ * Updates the value of the disk entry with a new value. This allows us to
+ * free up disk space in the non-backup case.
+ *
+ * @throws RegionClearedException
+ */
+ public static void update(DiskEntry entry, LocalRegion region, Object newValue, EntryEventImpl event) throws RegionClearedException {
+ DiskRegion dr = region.getDiskRegion();
+ if (newValue == null) {
+ throw new NullPointerException(LocalizedStrings.DiskEntry_ENTRYS_VALUE_SHOULD_NOT_BE_NULL.toLocalizedString());
+ }
+
+ //If we have concurrency checks enabled for a persistent region, we need
+ //to add an entry to the async queue for every update to maintain the RVV
+ boolean maintainRVV = region.concurrencyChecksEnabled && dr.isBackup();
+
+ Token oldValue = null;
+ int oldValueLength = 0;
+ boolean scheduleAsync = false;
+ boolean callRemoveFromDisk = false;
+ DiskId did = entry.getDiskId();
+ VersionTag tag = null;
+ Object syncObj = did;
+ if (syncObj == null) {
+ syncObj = entry;
+ }
+ if (syncObj == did) {
+ dr.acquireReadLock();
+ }
+ try {
+ synchronized (syncObj) {
+ oldValue = entry.getValueAsToken();
+ if (Token.isRemovedFromDisk(newValue)) {
+ if (dr.isBackup()) {
+ dr.testIsRecoveredAndClear(did); // fixes bug 41409
+ }
+ RuntimeException rte = null;
+ try {
+ if (!Token.isRemovedFromDisk(oldValue)) {
+ // removeFromDisk takes care of oldValueLength
+ if (dr.isSync()) {
+ removeFromDisk(entry, region, false);
+ } else {
+ callRemoveFromDisk = true; // do it outside the sync
+ }
+ }
+ } catch (RuntimeException e) {
+ rte = e;
+ throw e;
+ }
+ finally {
+ if (rte != null && (rte instanceof CacheClosedException)) {
+ // 47616: not to set the value to be removedFromDisk since it failed to persist
+ } else {
+ // Asif Ensure that the value is rightly set despite clear so
+ // that it can be distributed correctly
+ entry.setValueWithContext(region, newValue); // OFFHEAP newValue was already preparedForCache
+ }
+ }
+ }
+ else if (newValue instanceof RecoveredEntry) {
+ // Now that oplog creates are immediately put in cache
+ // a later oplog modify will get us here
+ RecoveredEntry re = (RecoveredEntry)newValue;
+ long oldKeyId = did.getKeyId();
+ long oldOplogId = did.getOplogId();
+ long newOplogId = re.getOplogId();
+ if (newOplogId != oldOplogId) {
+ did.setOplogId(newOplogId);
+ re.setOplogId(oldOplogId); // so caller knows oldoplog id
+ }
+ did.setOffsetInOplog(re.getOffsetInOplog());
+ // id already set
+ did.setUserBits(re.getUserBits());
+ oldValueLength = did.getValueLength();
+ did.setValueLength(re.getValueLength());
+ // The following undo and then do fixes bug 41849
+ // First, undo the stats done for the previous recovered value
+ if (oldKeyId < 0) {
+ dr.incNumOverflowOnDisk(-1L);
+ dr.incNumOverflowBytesOnDisk(-oldValueLength);
+ incrementBucketStats(region, 0/*InVM*/, -1/*OnDisk*/, -oldValueLength);
+ } else {
+ dr.incNumEntriesInVM(-1L);
+ incrementBucketStats(region, -1/*InVM*/, 0/*OnDisk*/, 0);
+ }
+ // Second, do the stats done for the current recovered value
+ if (re.getRecoveredKeyId() < 0) {
+ if (!entry.isValueNull()) {
+ try {
+ entry.handleValueOverflow(region);
+ entry.setValueWithContext(region, null); // fixes bug 41119
+ }finally {
+ entry.afterValueOverflow(region);
+ }
+
+ }
+ dr.incNumOverflowOnDisk(1L);
+ dr.incNumOverflowBytesOnDisk(did.getValueLength());
+ incrementBucketStats(region, 0/*InVM*/, 1/*OnDisk*/,
+ did.getValueLength());
+ } else {
+ entry.setValueWithContext(region, entry.prepareValueForCache(region, re.getValue(), false));
+ dr.incNumEntriesInVM(1L);
+ incrementBucketStats(region, 1/*InVM*/, 0/*OnDisk*/, 0);
+ }
+ }
+ else {
+ //The new value in the entry needs to be set after the disk writing
+ // has succeeded. If not , for GemFireXD , it is possible that other thread
+ // may pick this transient value from region entry ( which for
+ //offheap will eventually be released ) as index key,
+ //given that this operation is bound to fail in case of
+ //disk access exception.
+
+ //entry.setValueWithContext(region, newValue); // OFFHEAP newValue already prepared
+
+ if(did != null && did.isPendingAsync()) {
+ //if the entry was not yet written to disk, we didn't update
+ //the bytes on disk.
+ oldValueLength = 0;
+ } else {
+ oldValueLength = getValueLength(did);
+ }
+
+ if (dr.isBackup()) {
+ dr.testIsRecoveredAndClear(did); // fixes bug 41409
+ if (dr.isSync()) {
+ //In case of compression the value is being set first
+ // because atleast for now , GemFireXD does not support compression
+ // if and when it does support, this needs to be taken care of else
+ // we risk Bug 48965
+ if (AbstractRegionEntry.isCompressible(dr, newValue)) {
+ entry.setValueWithContext(region, newValue); // OFFHEAP newValue already prepared
+
+ // newValue is prepared and compressed. We can't write compressed values to disk.
+ writeToDisk(entry, region, false, event);
+ } else {
+ writeBytesToDisk(entry, region, false, createValueWrapper(newValue, event));
+ entry.setValueWithContext(region, newValue); // OFFHEAP newValue already prepared
+ }
+
+ } else if (did.isPendingAsync() && !maintainRVV) {
+ entry.setValueWithContext(region, newValue); // OFFHEAP newValue already prepared
+
+ // nothing needs to be done except
+ // fixing up LRU stats
+ // @todo fixup LRU stats if needed
+ // I'm not sure anything needs to be done here.
+ // If we have overflow and it decided to evict this entry
+ // how do we handle that case when we are async?
+ // Seems like the eviction code needs to leave the value
+ // in memory until the pendingAsync is done.
+ } else {
+ //if the entry is not async, we need to schedule it
+ //for regions with concurrency checks enabled, we add an entry
+ //to the queue for every entry.
+ scheduleAsync = true;
+ did.setPendingAsync(true);
+ VersionStamp stamp = entry.getVersionStamp();
+ if(stamp != null) {
+ tag = stamp.asVersionTag();
+ }
+ entry.setValueWithContext(region, newValue);
+ }
+ } else if (did != null) {
+ entry.setValueWithContext(region, newValue); // OFFHEAP newValue already prepared
+
+ // Mark the id as needing to be written
+ // The disk remove that this section used to do caused bug 30961
+ // @todo this seems wrong. How does leaving it on disk fix the bug?
+ did.markForWriting();
+ //did.setValueSerializedSize(0);
+ }else {
+ entry.setValueWithContext(region, newValue);
+ }
+
+ if (Token.isRemovedFromDisk(oldValue)) {
+ // Note we now initialize entries removed and then set their
+ // value once we find no existing entry.
+ // So this is the normal path for a brand new entry.
+ dr.incNumEntriesInVM(1L);
+ incrementBucketStats(region, 1/*InVM*/, 0/*OnDisk*/, 0);
+ }
+ }
+ if (entry instanceof LRUEntry) {
+ LRUEntry le = (LRUEntry)entry;
+ boolean wasEvicted = le.testEvicted();
+ le.unsetEvicted();
+ if (!Token.isRemovedFromDisk(newValue)) {
+ if (oldValue == null
+ // added null check for bug 41759
+ || wasEvicted && did != null && did.isPendingAsync()) {
+ // Note we do not append this entry because that will be
+ // done by lruEntryUpdate
+ dr.incNumEntriesInVM(1L);
+ dr.incNumOverflowOnDisk(-1L);
+ dr.incNumOverflowBytesOnDisk(-oldValueLength);
+ incrementBucketStats(region, 1/*InVM*/, -1/*OnDisk*/, -oldValueLength);
+ }
+ }
+ }
+ }
+ } finally {
+ if (syncObj == did) {
+ dr.releaseReadLock();
+ }
+ }
+ if (callRemoveFromDisk) {
+ removeFromDisk(entry, region, false, oldValue == null, false);
+ } else if (scheduleAsync && did.isPendingAsync()) {
+ // this needs to be done outside the above sync
+ scheduleAsyncWrite(new AsyncDiskEntry(region, entry, tag));
+ }
+ }
+
+ private static int getValueLength(DiskId did) {
+ int result = 0;
+ if (did != null) {
+ synchronized (did) {
+ result = did.getValueLength();
+ }
+ }
+ return result;
+ }
+
+ public static void updateRecoveredEntry(PlaceHolderDiskRegion drv,
+ DiskEntry entry,
+ RecoveredEntry newValue,RegionEntryContext context)
+ {
+ if (newValue == null) {
+ throw new NullPointerException(LocalizedStrings.DiskEntry_ENTRYS_VALUE_SHOULD_NOT_BE_NULL.toLocalizedString());
+ }
+ DiskId did = entry.getDiskId();
+ synchronized (did) {
+ boolean oldValueWasNull = entry.isValueNull();
+ int oldValueLength = did.getValueLength();
+ // Now that oplog creates are immediately put in cache
+ // a later oplog modify will get us here
+ long oldOplogId = did.getOplogId();
+ long newOplogId = newValue.getOplogId();
+ if (newOplogId != oldOplogId) {
+ did.setOplogId(newOplogId);
+ newValue.setOplogId(oldOplogId); // so caller knows oldoplog id
+ }
+ did.setOffsetInOplog(newValue.getOffsetInOplog());
+ // id already set
+ did.setUserBits(newValue.getUserBits());
+ did.setValueLength(newValue.getValueLength());
+ if (newValue.getRecoveredKeyId() >= 0) {
+ entry.setValueWithContext(context, entry.prepareValueForCache(drv, newValue.getValue(),
+ false));
+ } else {
+ if (!oldValueWasNull) {
+ try {
+ entry.handleValueOverflow(context);
+ entry.setValueWithContext(context,null); // fixes bug 41119
+ }finally {
+ entry.afterValueOverflow(context);
+ }
+ }
+ }
+ if (entry instanceof LRUEntry) {
+ LRUEntry le = (LRUEntry)entry;
+ assert !le.testEvicted();
+ // we don't allow eviction during recovery
+ if (oldValueWasNull) {
+ // Note we do not append this entry because that will be
+ // done by lruEntryUpdate
+ drv.incNumEntriesInVM(1L);
+ drv.incNumOverflowOnDisk(-1L);
+ drv.incNumOverflowBytesOnDisk(-oldValueLength);
+ //No need to call incrementBucketStats here because we don't have
+ //a real bucket region, this is during recovery from disk.
+ }
+ }
+ }
+ }
+
+ public static Object getValueInVMOrDiskWithoutFaultIn(DiskEntry entry, LocalRegion region) {
+ Object result = OffHeapHelper.copyAndReleaseIfNeeded(getValueOffHeapOrDiskWithoutFaultIn(entry, region));
+ if (result instanceof CachedDeserializable) {
+ result = ((CachedDeserializable)result).getDeserializedValue(null, null);
+ }
+ if (result instanceof StoredObject) {
+ ((StoredObject) result).release();
+ throw new IllegalStateException("sqlf tried to use getValueInVMOrDiskWithoutFaultIn");
+ }
+ return result;
+ }
+
+ @Retained
+ public static Object getValueOffHeapOrDiskWithoutFaultIn(DiskEntry entry, LocalRegion region) {
+ @Retained Object v = entry._getValueRetain(region, true); // TODO:KIRK:OK Object v = entry.getValueWithContext(region);
+ if (v == null || Token.isRemovedFromDisk(v)
+ && !region.isIndexCreationThread()) {
+ synchronized (entry) {
+ v = entry._getValueRetain(region, true); // TODO:KIRK:OK v = entry.getValueWithContext(region);
+ if (v == null) {
+ v = Helper.getOffHeapValueOnDiskOrBuffer(entry, region.getDiskRegion(),region);
+ }
+ }
+ }
+ if (Token.isRemovedFromDisk(v)) {
+ // fix for bug 31800
+ v = null;
+// } else if (v instanceof ByteSource) {
+// // If the ByteSource contains a Delta or ListOfDelta then we want to deserialize it
+// Object deserVal = ((CachedDeserializable)v).getDeserializedForReading();
+// if (deserVal != v) {
+// OffHeapHelper.release(v);
+// v = deserVal;
+// }
+ }
+ return v;
+ }
+
+ /**
+ *
+ * @param entry
+ * @param region
+ * @return Value
+ * @throws DiskAccessException
+ */
+ public static Object faultInValue(DiskEntry entry, LocalRegion region) {
+ return faultInValue(entry, region, false);
+ }
+ @Retained
+ public static Object faultInValueRetain(DiskEntry entry, LocalRegion region) {
+ return faultInValue(entry, region, true);
+ }
+ /**
+ * @param retainResult if true then the result may be a retained off-heap reference
+ */
+ @Retained
+ private static Object faultInValue(DiskEntry entry, LocalRegion region, boolean retainResult)
+ {
+ DiskRegion dr = region.getDiskRegion();
+ @Retained Object v = entry._getValueRetain(region, true); // TODO:KIRK:OK Object v = entry.getValueWithContext(region);
+ boolean lruFaultedIn = false;
+ boolean done = false;
+ try {
+ //Asif: If the entry is instance of LRU then DidkRegion cannot be null.
+ //Since SqlFabric is accessing this method direcly & it passes the owning region,
+ //if the region happens to be persistent PR type, the owning region passed is PR,
+ // but it will have DiskRegion as null. SqlFabric takes care of passing owning region
+ // as BucketRegion in case of Overflow type entry. This is fix for Bug # 41804
+ if ( entry instanceof LRUEntry && !dr.isSync() ) {
+ synchronized (entry) {
+ DiskId did = entry.getDiskId();
+ if (did != null && did.isPendingAsync()) {
+ done = true;
+ // See if it is pending async because of a faultOut.
+ // If so then if we are not a backup then we can unschedule the pending async.
+ // In either case we need to do the lruFaultIn logic.
+ boolean evicted = ((LRUEntry)entry).testEvicted();
+ if (evicted) {
+ if (!dr.isBackup()) {
+ // @todo do we also need a bit that tells us if it is in the async queue?
+ // Seems like we could end up adding it to the queue multiple times.
+ did.setPendingAsync(false);
+ }
+ // since it was evicted fix the stats here
+ dr.incNumEntriesInVM(1L);
+ dr.incNumOverflowOnDisk(-1L);
+ // no need to dec overflowBytesOnDisk because it was not inced in this case.
+ incrementBucketStats(region, 1/*InVM*/, -1/*OnDisk*/, 0);
+ }
+ lruEntryFaultIn((LRUEntry) entry, region);
+ lruFaultedIn = true;
+ }
+ }
+ }
+ if (!done
+ && (v == null || Token.isRemovedFromDisk(v) && !region.isIndexCreationThread())) {
+ synchronized (entry) {
+ v = entry._getValueRetain(region, true); // TODO:KIRK:OK v = entry.getValueWithContext(region);
+ if (v == null) {
+ v = readValueFromDisk(entry, region);
+ if (entry instanceof LRUEntry) {
+ if (v != null && !Token.isInvalid(v)) {
+ lruEntryFaultIn((LRUEntry) entry, region);
+
+ lruFaultedIn = true;
+ }
+ }
+ }
+ }
+ }
+ } finally {
+ if (!retainResult) {
+ v = OffHeapHelper.copyAndReleaseIfNeeded(v);
+ // At this point v should be either a heap object
+ }
+ }
+ if (Token.isRemoved(v)) {
+ // fix for bug 31800
+ v = null;
+ } else {
+ ((RegionEntry)entry).setRecentlyUsed();
+ }
+ if (lruFaultedIn) {
+ lruUpdateCallback(region);
+ }
+ return v; // OFFHEAP: the value ends up being returned by RegionEntry.getValue
+ }
+
+ public static void recoverValue(DiskEntry entry, long oplogId, DiskRecoveryStore recoveryStore, ByteArrayDataInput in) {
+ boolean lruFaultedIn = false;
+ synchronized (entry) {
+ if (entry.isValueNull()) {
+ DiskId did = entry.getDiskId();
+ if (did != null) {
+ Object value = null;
+ DiskRecoveryStore region = recoveryStore;
+ DiskRegionView dr = region.getDiskRegionView();
+ dr.acquireReadLock();
+ try {
+ synchronized (did) {
+ // don't read if the oplog has changed.
+ if (oplogId == did.getOplogId()) {
+ value = getValueFromDisk(dr, did, in);
+ if (value != null) {
+ setValueOnFaultIn(value, did, entry, dr, region);
+ }
+ }
+ }
+ } finally {
+ dr.releaseReadLock();
+ }
+ if (entry instanceof LRUEntry) {
+ if (value != null && !Token.isInvalid(value)) {
+ lruEntryFaultIn((LRUEntry) entry, recoveryStore);
+ lruFaultedIn = true;
+ }
+ }
+ }
+ }
+ }
+ if (lruFaultedIn) {
+ lruUpdateCallback(recoveryStore);
+ }
+ }
+
+ /**
+ * Caller must have "did" synced.
+ */
+ private static Object getValueFromDisk(DiskRegionView dr, DiskId did, ByteArrayDataInput in) {
+ Object value;
+ if (dr.isBackup() && did.getKeyId() == DiskRegion.INVALID_ID) {
+ // must have been destroyed
+ value = null;
+ } else {
+ if (did.isKeyIdNegative()) {
+ did.setKeyId(- did.getKeyId());
+ }
+ // if a bucket region then create a CachedDeserializable here instead of object
+ value = dr.getRaw(did); // fix bug 40192
+ if (value instanceof BytesAndBits) {
+ BytesAndBits bb = (BytesAndBits)value;
+ if (EntryBits.isInvalid(bb.getBits())) {
+ value = Token.INVALID;
+ } else if (EntryBits.isLocalInvalid(bb.getBits())) {
+ value = Token.LOCAL_INVALID;
+ } else if (EntryBits.isTombstone(bb.getBits())) {
+ value = Token.TOMBSTONE;
+ } else if (EntryBits.isSerialized(bb.getBits())) {
+ value = readSerializedValue(bb.getBytes(), bb.getVersion(), in, false);
+ } else {
+ value = readRawValue(bb.getBytes(), bb.getVersion(), in);
+ }
+ }
+ }
+ return value;
+ }
+
+ private static void lruUpdateCallback(DiskRecoveryStore recoveryStore) {
+ /*
+ * Used conditional check to see if
+ * if its a LIFO Enabled,
+ * yes then disable lruUpdateCallback()
+ * and called updateStats()
+ * its keep track of actual entries
+ * present in memory - useful when
+ * checking capacity constraint
+ */
+ try {
+ if (recoveryStore.getEvictionAttributes() != null
+ && recoveryStore.getEvictionAttributes().getAlgorithm().isLIFO()) {
+ ((VMLRURegionMap) recoveryStore.getRegionMap()).updateStats();
+ return;
+ }
+ // this must be done after releasing synchronization
+ recoveryStore.getRegionMap().lruUpdateCallback();
+ }catch( DiskAccessException dae) {
+ recoveryStore.handleDiskAccessException(dae);
+ throw dae;
+ }
+ }
+
+ private static void lruEntryFaultIn(LRUEntry entry, DiskRecoveryStore recoveryStore) {
+ RegionMap rm = (RegionMap)recoveryStore.getRegionMap();
+ try {
+ rm.lruEntryFaultIn((LRUEntry) entry);
+ }catch(DiskAccessException dae) {
+ recoveryStore.handleDiskAccessException(dae);
+ throw dae;
+ }
+ }
+
+ /**
+ * Returns the value of this map entry, reading it from disk, if necessary.
+ * Sets the value in the entry.
+ * This is only called by the faultIn code once it has determined that
+ * the value is no longer in memory.
+ * return the result will only be off-heap if the value is a sqlf ByteSource. Otherwise result will be on-heap.
+ * Caller must have "entry" synced.
+ */
+ @Retained
+ private static Object readValueFromDisk(DiskEntry entry, DiskRecoveryStore region) {
+
+ DiskRegionView dr = region.getDiskRegionView();
+ DiskId did = entry.getDiskId();
+ if (did == null) {
+ return null;
+ }
+ dr.acquireReadLock();
+ try {
+ synchronized (did) {
+ Object value = getValueFromDisk(dr, did, null);
+ if (value == null) return null;
+ @Unretained Object preparedValue = setValueOnFaultIn(value, did, entry, dr, region);
+ // For Sqlfire we want to return the offheap representation.
+ // So we need to retain it for the caller to release.
+ /*if (preparedValue instanceof ByteSource) {
+ // This is the only case in which we return a retained off-heap ref.
+ ((ByteSource)preparedValue).retain();
+ return preparedValue;
+ } else */{
+ return value;
+ }
+ }
+ } finally {
+ dr.releaseReadLock();
+ }
+ }
+
+ /**
+ * Caller must have "entry" and "did" synced and "dr" readLocked.
+ * @return the unretained result must be used by the caller before it releases the sync on "entry".
+ */
+ @Unretained
+ private static Object setValueOnFaultIn(Object value, DiskId did, DiskEntry entry, DiskRegionView dr, DiskRecoveryStore region) {
+// dr.getOwner().getCache().getLogger().info("DEBUG: faulting in entry with key " + entry.getKey());
+ int bytesOnDisk = getValueLength(did);
+ // Retained by the prepareValueForCache call for the region entry.
+ // NOTE that we return this value unretained because the retain is owned by the region entry not the caller.
+ @Retained Object preparedValue = entry.prepareValueForCache((RegionEntryContext) region, value,
+ false);
+ region.updateSizeOnFaultIn(entry.getKey(), region.calculateValueSize(preparedValue), bytesOnDisk);
+ //did.setValueSerializedSize(0);
+ // I think the following assertion is true but need to run
+ // a regression with it. Reenable this post 6.5
+ //Assert.assertTrue(entry._getValue() == null);
+ entry.setValueWithContext((RegionEntryContext) region, preparedValue);
+ dr.incNumEntriesInVM(1L);
+ dr.incNumOverflowOnDisk(-1L);
+ dr.incNumOverflowBytesOnDisk(-bytesOnDisk);
+ incrementBucketStats(region, 1/*InVM*/, -1/*OnDisk*/, -bytesOnDisk);
+ return preparedValue;
+ }
+
+ static Object readSerializedValue(byte[] valueBytes, Version version,
+ ByteArrayDataInput in, boolean forceDeserialize) {
+ if (forceDeserialize) {
+ // deserialize checking for product version change
+ return EntryEventImpl.deserialize(valueBytes, version, in);
+ }
+ else {
+ // TODO: upgrades: is there a case where GemFire values are internal
+ // ones that need to be upgraded transparently; probably messages
+ // being persisted (gateway events?)
+ return CachedDeserializableFactory.create(valueBytes);
+ }
+ }
+
+ static Object readRawValue(byte[] valueBytes, Version version,
+ ByteArrayDataInput in) {
+ /*
+ final StaticSystemCallbacks sysCb;
+ if (version != null && (sysCb = GemFireCacheImpl.FactoryStatics
+ .systemCallbacks) != null) {
+ // may need to change serialized shape for SQLFire
+ return sysCb.fromVersion(valueBytes, false, version, in);
+ }
+ else */ {
+ return valueBytes;
+ }
+ }
+
+ public static void incrementBucketStats(Object owner,
+ int entriesInVmDelta,
+ int overflowOnDiskDelta,
+ int overflowBytesOnDiskDelta) {
+ if (owner instanceof BucketRegion) {
+ ((BucketRegion)owner).incNumEntriesInVM(entriesInVmDelta);
+ ((BucketRegion)owner).incNumOverflowOnDisk(overflowOnDiskDelta);
+ ((BucketRegion)owner).incNumOverflowBytesOnDisk(overflowBytesOnDiskDelta);
+ } else if (owner instanceof DiskRegionView) {
+ ((DiskRegionView)owner).incNumOverflowBytesOnDisk(overflowBytesOnDiskDelta);
+ }
+ }
+
+ /**
+ * Writes the value of this DiskEntry to disk and
+ * null s out the reference to the value to free up VM space.
+ *