Static memory snapshot collecting and restoring

bootstrap/src/sun/nio/ch/lincheck/EventTracker.java

@@ -32,6 +32,7 @@ public interface EventTracker {
     void beforeNewObjectCreation(String className);
     void afterNewObjectCreation(Object obj);
 
+    void updateSnapshotWithEagerTracking(Object[] objs);
     boolean beforeReadField(Object obj, String className, String fieldName, int codeLocation,
                             boolean isStatic, boolean isFinal);
     boolean beforeReadArrayElement(Object array, int index, int codeLocation);

bootstrap/src/sun/nio/ch/lincheck/Injections.java

@@ -294,6 +294,13 @@ public static void afterNewObjectCreation(Object obj) {
         getEventTracker().afterNewObjectCreation(obj);
     }
 
+    /**
+     * Called from instrumented code on constructor invocation, where passed objects are subtypes of the constructor class type.
+     */
+    public static void updateSnapshotWithEagerTracking(Object[] objs) {
+        getEventTracker().updateSnapshotWithEagerTracking(objs);
+    }
+
     /**
      * Called from the instrumented code to replace [java.lang.Object.hashCode] method call with some
      * deterministic value.

src/jvm/main/org/jetbrains/kotlinx/lincheck/ObjectTraverser.kt

@@ -22,7 +22,7 @@ package org.jetbrains.kotlinx.lincheck
 
 import org.jetbrains.kotlinx.lincheck.strategy.managed.ObjectLabelFactory.getObjectNumber
 import org.jetbrains.kotlinx.lincheck.util.*
-import java.lang.reflect.Modifier
+import org.jetbrains.kotlinx.lincheck.util.readFieldViaUnsafe
 import java.math.BigDecimal
 import java.math.BigInteger
 import kotlin.coroutines.Continuation

src/jvm/main/org/jetbrains/kotlinx/lincheck/Utils.kt

@@ -10,13 +10,12 @@
 package org.jetbrains.kotlinx.lincheck
 
 import kotlinx.coroutines.*
-import sun.nio.ch.lincheck.*
 import org.jetbrains.kotlinx.lincheck.runner.*
 import org.jetbrains.kotlinx.lincheck.strategy.managed.*
 import org.jetbrains.kotlinx.lincheck.transformation.LincheckClassFileTransformer
-import org.jetbrains.kotlinx.lincheck.util.UnsafeHolder
+import org.jetbrains.kotlinx.lincheck.util.readFieldViaUnsafe
 import org.jetbrains.kotlinx.lincheck.verifier.*
-import org.jetbrains.kotlinx.lincheck.util.*
+import sun.nio.ch.lincheck.*
 import java.io.PrintWriter
 import java.io.StringWriter
 import java.lang.ref.*
@@ -213,21 +212,63 @@ internal val Throwable.text: String get() {
     return writer.buffer.toString()
 }
 
+/**
+ * Returns all found fields in the hierarchy.
+ * Multiple fields with the same name and the same type may be returned
+ * if they appear in the subclass and a parent class.
+ */
+internal val Class<*>.allDeclaredFieldWithSuperclasses get(): List<Field> {
+    val fields: MutableList<Field> = ArrayList<Field>()
+    var currentClass: Class<*>? = this
+    while (currentClass != null) {
+        val declaredFields: Array<Field> = currentClass.declaredFields
+        fields.addAll(declaredFields)
+        currentClass = currentClass.superclass
+    }
+    return fields
+}
 
-@Suppress("DEPRECATION")
-internal fun findFieldNameByOffset(targetType: Class<*>, offset: Long): String? {
-    // Extract the private offset value and find the matching field.
-    for (field in targetType.declaredFields) {
+/**
+ * Finds a public/protected/private/internal field in the class and its superclasses/interfaces by name.
+ *
+ * @param fieldName the name of the field to find.
+ * @return the [java.lang.reflect.Field] object if found, or `null` if not found.
+ */
+internal fun Class<*>.findField(fieldName: String): Field {
+    // Search in the class hierarchy
+    var clazz: Class<*>? = this
+    while (clazz != null) {
+        // Check class itself
         try {
-            if (Modifier.isNative(field.modifiers)) continue
-            val fieldOffset = if (Modifier.isStatic(field.modifiers)) UnsafeHolder.UNSAFE.staticFieldOffset(field)
-            else UnsafeHolder.UNSAFE.objectFieldOffset(field)
-            if (fieldOffset == offset) return field.name
-        } catch (t: Throwable) {
-            t.printStackTrace()
+            return clazz.getDeclaredField(fieldName)
+        }
+        catch (_: NoSuchFieldException) {}
+
+        // Check interfaces
+        for (interfaceClass in clazz.interfaces) {
+            try {
+                return interfaceClass.getDeclaredField(fieldName)
+            }
+            catch (_: NoSuchFieldException) {}
         }
+
+        // Move up the hierarchy
+        clazz = clazz.superclass
     }
-    return null // Field not found
+
+    throw NoSuchFieldException("Class '${this.name}' does not have field '$fieldName'")
+}
+
+/**
+ * Reads a [field] of the owner object [obj] via Unsafe, and falls back onto the reflexivity in case of failure.
+ */
+internal fun readFieldSafely(obj: Any?, field: Field): kotlin.Result<Any?> {
+    // we wrap an unsafe read into `runCatching` to handle `UnsupportedOperationException`,
+    // which can be thrown, for instance, when attempting to read
+    // a field of a hidden or record class (starting from Java 15);
+    // in this case we fall back to read via reflection
+    return runCatching { readFieldViaUnsafe(obj, field) }
+        .recoverCatching { field.apply { isAccessible = true }.get(obj) }
 }
 
 /**

src/jvm/main/org/jetbrains/kotlinx/lincheck/strategy/Strategy.kt

@@ -11,6 +11,7 @@ package org.jetbrains.kotlinx.lincheck.strategy
 
 import org.jetbrains.kotlinx.lincheck.runner.*
 import org.jetbrains.kotlinx.lincheck.execution.ExecutionScenario
+import org.jetbrains.kotlinx.lincheck.strategy.managed.ManagedStrategy
 import org.jetbrains.kotlinx.lincheck.strategy.managed.Trace
 import org.jetbrains.kotlinx.lincheck.verifier.Verifier
 import java.io.Closeable
@@ -101,15 +102,25 @@ abstract class Strategy protected constructor(
  * @return the failure, if detected, null otherwise.
  */
 fun Strategy.runIteration(invocations: Int, verifier: Verifier): LincheckFailure? {
+    var failure: LincheckFailure? = null
+
     for (invocation in 0 until invocations) {
         if (!nextInvocation())
-            return null
+            break
         val result = runInvocation()
-        val failure = verify(result, verifier)
+        try {
+            failure = verify(result, verifier)
+        }
+        finally {
+            if (this is ManagedStrategy) {
+                restoreStaticMemorySnapshot()
+            }
+        }
         if (failure != null)
-            return failure
+            break
     }
-    return null
+
+    return failure
 }
 
 /**

src/jvm/main/org/jetbrains/kotlinx/lincheck/strategy/managed/AtomicFieldUpdaterNames.kt

@@ -10,8 +10,8 @@
 
 package org.jetbrains.kotlinx.lincheck.strategy.managed
 
-import org.jetbrains.kotlinx.lincheck.findFieldNameByOffset
 import org.jetbrains.kotlinx.lincheck.util.UnsafeHolder.UNSAFE
+import org.jetbrains.kotlinx.lincheck.util.findFieldNameByOffset
 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater
 import java.util.concurrent.atomic.AtomicLongFieldUpdater
 import java.util.concurrent.atomic.AtomicReferenceFieldUpdater

src/jvm/main/org/jetbrains/kotlinx/lincheck/strategy/managed/FieldSearchHelper.kt

@@ -57,7 +57,7 @@ internal object FieldSearchHelper {
         traverseObjectGraph(
             testObject,
             traverseStaticFields = true,
-            onArrayElement = { _, _, _ -> null }, // do not traverse array elements further
+            onArrayElement = null, // do not traverse array elements further
             onField = { ownerObject, field, fieldValue ->
                 when {
                     field.type.isPrimitive || fieldValue == null -> null

src/jvm/main/org/jetbrains/kotlinx/lincheck/strategy/managed/ManagedCTestConfiguration.kt

@@ -52,6 +52,7 @@ abstract class ManagedCTestConfiguration(
         const val DEFAULT_CHECK_OBSTRUCTION_FREEDOM = false
         const val DEFAULT_HANGING_DETECTION_THRESHOLD = 101
         const val LIVELOCK_EVENTS_THRESHOLD = 10001
+        const val DEFAULT_RESTORE_STATIC_MEMORY = true
         val DEFAULT_GUARANTEES = listOf<ManagedStrategyGuarantee>()
     }
 }

src/jvm/main/org/jetbrains/kotlinx/lincheck/strategy/managed/ManagedOptions.kt

@@ -14,6 +14,7 @@ import org.jetbrains.kotlinx.lincheck.strategy.managed.ManagedCTestConfiguration
 import org.jetbrains.kotlinx.lincheck.strategy.managed.ManagedCTestConfiguration.Companion.DEFAULT_GUARANTEES
 import org.jetbrains.kotlinx.lincheck.strategy.managed.ManagedCTestConfiguration.Companion.DEFAULT_HANGING_DETECTION_THRESHOLD
 import org.jetbrains.kotlinx.lincheck.strategy.managed.ManagedCTestConfiguration.Companion.DEFAULT_INVOCATIONS
+import org.jetbrains.kotlinx.lincheck.strategy.managed.ManagedCTestConfiguration.Companion.DEFAULT_RESTORE_STATIC_MEMORY
 import java.util.*
 
 /**
@@ -23,6 +24,7 @@ abstract class ManagedOptions<OPT : Options<OPT, CTEST>, CTEST : CTestConfigurat
     protected var invocationsPerIteration = DEFAULT_INVOCATIONS
     protected var checkObstructionFreedom = DEFAULT_CHECK_OBSTRUCTION_FREEDOM
     protected var hangingDetectionThreshold = DEFAULT_HANGING_DETECTION_THRESHOLD
+    protected var restoreStaticMemory = DEFAULT_RESTORE_STATIC_MEMORY
     protected val guarantees: MutableList<ManagedStrategyGuarantee> = ArrayList(DEFAULT_GUARANTEES)
 
     /**
@@ -50,6 +52,13 @@ abstract class ManagedOptions<OPT : Options<OPT, CTEST>, CTEST : CTestConfigurat
         this.hangingDetectionThreshold = hangingDetectionThreshold
     }
 
+    /**
+     * Set to `true` to activate the static memory snapshot tracking and restoring algorithm.
+     */
+    fun restoreStaticMemory(restoreStaticMemory: Boolean): OPT = applyAndCast {
+        this.restoreStaticMemory = restoreStaticMemory
+    }
+
     /**
      * Add a guarantee that methods in some classes are either correct in terms of concurrent execution or irrelevant.
      * These guarantees can be used for optimization. For example, we can add a guarantee that all the methods

src/jvm/main/org/jetbrains/kotlinx/lincheck/strategy/managed/ManagedStrategy.kt

@@ -82,6 +82,9 @@ abstract class ManagedStrategy(
     // Tracker of the thread parking.
     protected abstract val parkingTracker: ParkingTracker
 
+    // Snapshot of the memory, reachable from static fields
+    protected val staticMemorySnapshot = SnapshotTracker()
+
     // InvocationResult that was observed by the strategy during the execution (e.g., a deadlock).
     @Volatile
     protected var suddenInvocationResult: InvocationResult? = null
@@ -216,13 +219,26 @@ abstract class ManagedStrategy(
         parkingTracker.reset()
     }
 
+    /**
+     * Restores recorded values of all memory reachable from static state.
+     */
+    fun restoreStaticMemorySnapshot() {
+        staticMemorySnapshot.restoreValues()
+    }
+
     /**
      * Runs the current invocation.
      */
     override fun runInvocation(): InvocationResult {
         while (true) {
             initializeInvocation()
-            val result = runner.run()
+            val result: InvocationResult
+            try {
+                result = runner.run()
+            }
+            finally {
+                restoreStaticMemorySnapshot()
+            }
             // In case the runner detects a deadlock, some threads can still manipulate the current strategy,
             // so we're not interested in suddenInvocationResult in this case
             // and immediately return RunnerTimeoutInvocationResult.
@@ -742,6 +758,7 @@ abstract class ManagedStrategy(
      */
     override fun beforeReadField(obj: Any?, className: String, fieldName: String, codeLocation: Int,
                                  isStatic: Boolean, isFinal: Boolean) = runInIgnoredSection {
+         updateSnapshotOnFieldAccess(obj, className.canonicalClassName, fieldName)
         // We need to ensure all the classes related to the reading object are instrumented.
         // The following call checks all the static fields.
         if (isStatic) {
@@ -778,6 +795,7 @@ abstract class ManagedStrategy(
 
     /** Returns <code>true</code> if a switch point is created. */
     override fun beforeReadArrayElement(array: Any, index: Int, codeLocation: Int): Boolean = runInIgnoredSection {
+        updateSnapshotOnArrayElementAccess(array, index)
         if (!objectTracker.shouldTrackObjectAccess(array)) {
             return@runInIgnoredSection false
         }
@@ -813,6 +831,7 @@ abstract class ManagedStrategy(
 
     override fun beforeWriteField(obj: Any?, className: String, fieldName: String, value: Any?, codeLocation: Int,
                                   isStatic: Boolean, isFinal: Boolean): Boolean = runInIgnoredSection {
+        updateSnapshotOnFieldAccess(obj, className.canonicalClassName, fieldName)
         objectTracker.registerObjectLink(fromObject = obj ?: StaticObject, toObject = value)
         if (!objectTracker.shouldTrackObjectAccess(obj ?: StaticObject)) {
             return@runInIgnoredSection false
@@ -842,6 +861,7 @@ abstract class ManagedStrategy(
     }
 
     override fun beforeWriteArrayElement(array: Any, index: Int, value: Any?, codeLocation: Int): Boolean = runInIgnoredSection {
+        updateSnapshotOnArrayElementAccess(array, index)
         objectTracker.registerObjectLink(fromObject = array, toObject = value)
         if (!objectTracker.shouldTrackObjectAccess(array)) {
             return@runInIgnoredSection false
@@ -909,6 +929,29 @@ abstract class ManagedStrategy(
         }
     }
 
+    /**
+     * Tracks a specific field of an [obj], if the [obj] is either `null` (which means that field is static),
+     * or one this objects which contains it is already stored.
+     */
+    fun updateSnapshotOnFieldAccess(obj: Any?, className: String, fieldName: String) = runInIgnoredSection {
+        staticMemorySnapshot.trackField(obj, className, fieldName)
+    }
+
+    /**
+     * Tracks a specific [array] element at [index], if the [array] is already tracked.
+     */
+    fun updateSnapshotOnArrayElementAccess(array: Any, index: Int) = runInIgnoredSection {
+        staticMemorySnapshot.trackArrayCell(array, index)
+    }
+
+    /**
+     * Tracks all objects in [objs] eagerly.
+     * Required as a trick to overcome issue with leaking this in constructors, see https://github.com/JetBrains/lincheck/issues/424.
+     */
+    override fun updateSnapshotWithEagerTracking(objs: Array<Any?>) = runInIgnoredSection {
+        staticMemorySnapshot.trackObjects(objs)
+    }
+
     private fun methodGuaranteeType(owner: Any?, className: String, methodName: String): ManagedGuaranteeType? = runInIgnoredSection {
         userDefinedGuarantees?.forEach { guarantee ->
             val ownerName = owner?.javaClass?.canonicalName ?: className