Skip to content

Commit

Permalink
Exceptions provided to the plugin (#304)
Browse files Browse the repository at this point in the history 
Exceptions provided to plugin via testFailed method
  • Loading branch information
@avpotapov00
avpotapov00 authored Apr 23, 2024
1 parent 181bbe9 commit 07e2798
Show file tree
Hide file tree
Showing 5 changed files with 131 additions and 27 deletions.
6 changes: 4 additions & 2 deletions src/jvm/main/org/jetbrains/kotlinx/lincheck/IdeaPlugin.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,18 +27,20 @@ const val MINIMAL_PLUGIN_VERSION = "0.2"
* then [beforeEvent] method is called on each trace point.
*
* @param failureType string representation of the failure type.
* (`INCORRECT_RESULTS`, `OBSTRUCTION_FREEDOM_VIOLATION`, `UNEXPECTED_EXCEPTION`, `VALIDATION_FAILURE`, `DEADLOCK`).
* (`INCORRECT_RESULTS`, `OBSTRUCTION_FREEDOM_VIOLATION`, `UNEXPECTED_EXCEPTION`, `VALIDATION_FAILURE`, `DEADLOCK` or `INTERNAL_BUG`).
* @param trace failed test trace, where each trace point is represented as a string
* (because it's the easiest way to provide some information to the debugger).
* @param version current Lincheck version
* @param minimalPluginVersion minimal compatible plugin version
* @param exceptions representation of the exceptions with their stacktrace occurred during the execution
*/
@Suppress("UNUSED_PARAMETER")
fun testFailed(
failureType: String,
trace: Array<String>,
version: String?,
minimalPluginVersion: String
minimalPluginVersion: String,
exceptions: Array<String>
) {
}

Expand Down
8 changes: 8 additions & 0 deletions src/jvm/main/org/jetbrains/kotlinx/lincheck/Utils.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ import org.jetbrains.kotlinx.lincheck.runner.*
import org.jetbrains.kotlinx.lincheck.strategy.managed.*
import org.jetbrains.kotlinx.lincheck.transformation.LincheckClassFileTransformer
import org.jetbrains.kotlinx.lincheck.verifier.*
import java.io.PrintWriter
import java.io.StringWriter
import java.lang.ref.*
import java.lang.reflect.*
import java.lang.reflect.Method
Expand Down Expand Up @@ -186,6 +188,12 @@ internal fun exceptionCanBeValidExecutionResult(exception: Throwable): Boolean {
exception !is ForcibleExecutionFinishError
}

internal val Throwable.text: String get() {
val writer = StringWriter()
printStackTrace(PrintWriter(writer))
return writer.buffer.toString()
}

/**
* Utility exception for test purposes.
* When this exception is thrown by an operation, it will halt testing with [UnexpectedExceptionInvocationResult].
Expand Down
63 changes: 54 additions & 9 deletions src/jvm/main/org/jetbrains/kotlinx/lincheck/strategy/managed/TraceReporter.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,7 @@ internal fun constructTraceGraph(
trace: Trace,
exceptionStackTraces: Map<Throwable, ExceptionNumberAndStacktrace>
): List<TraceNode> {
val resultProvider = ExecutionResultsProvider(results, failure)
val scenario = failure.scenario
val tracePoints = trace.trace
// last events that were executed for each thread. It is either thread finish events or events before crash
Expand Down Expand Up @@ -176,7 +177,7 @@ internal fun constructTraceGraph(
last = lastNode,
callDepth = 0,
actorRepresentation = actorRepresentations[iThread][nextActor],
resultRepresentation = results[iThread, nextActor]
resultRepresentation = resultProvider[iThread, nextActor]
?.let { actorNodeResultRepresentation(it, exceptionStackTraces) }
)
}
Expand Down Expand Up @@ -209,7 +210,7 @@ internal fun constructTraceGraph(
for (iThread in actorNodes.indices) {
for (actorId in actorNodes[iThread].indices) {
var actorNode = actorNodes[iThread][actorId]
val actorResult = results[iThread, actorId]
val actorResult = resultProvider[iThread, actorId]
// in case of empty trace, we want to show at least the actor nodes themselves;
// however, no actor nodes will be created by the code above, so we need to create them explicitly here.
if (actorNode == null && actorResult != null) {
Expand All @@ -229,9 +230,15 @@ internal fun constructTraceGraph(
// insert an ActorResultNode between the last actor event and the next event after it
val lastEvent = actorNode.lastInternalEvent
val lastEventNext = lastEvent.next
val result = results[iThread, actorId]
val result = resultProvider[iThread, actorId]
val resultRepresentation = result?.let { resultRepresentation(result, exceptionStackTraces) }
val resultNode = ActorResultNode(iThread, lastEvent, actorNode.callDepth + 1, resultRepresentation)
val resultNode = ActorResultNode(
iThread = iThread,
last = lastEvent,
callDepth = actorNode.callDepth + 1,
resultRepresentation = resultRepresentation,
exceptionNumberIfExceptionResult = if (result is ExceptionResult) exceptionStackTraces[result.throwable]?.number else null
)
actorNode.addInternalEvent(resultNode)
resultNode.next = lastEventNext
}
Expand All @@ -244,6 +251,43 @@ internal fun constructTraceGraph(
return traceGraphNodesSections.map { it.first() }
}

/**
* Helper class to provider execution results, including a validation function result
*/
private class ExecutionResultsProvider(result: ExecutionResult?, failure: LincheckFailure) {

/**
* A map of type Map<(threadId, actorId) -> Result>
*/
private val threadNumberToActorResultMap: Map<Pair<Int, Int>, Result> = when {
// If the results of the failure are present, then just collect them to a map.
// In that case, we know that the failure reason is not validation function, so we ignore it.
(result != null) -> {
result.threadsResults
.flatMapIndexed { tId, actors -> actors.flatMapIndexed { actorId, result ->
listOf((tId to actorId) to result)
}}
.toMap()
}

// If validation function is the reason if the failure then the only result we're interested in
// is the validation function exception.
failure is ValidationFailure -> {
mapOf((0 to firstThreadActorCount(failure)) to ExceptionResult.create(failure.exception, false))
}

else -> emptyMap()
}

operator fun get(iThread: Int, actorId: Int): Result? {
return threadNumberToActorResultMap[iThread to actorId]
}

private fun firstThreadActorCount(failure: ValidationFailure): Int =
failure.scenario.initExecution.size + failure.scenario.parallelExecution[0].size + failure.scenario.postExecution.size

}

/**
* Creates united actors representation, including invoked actors and validation functions.
* In output construction, we treat validation function call like a regular actor for unification.
Expand All @@ -257,17 +301,14 @@ private fun createActorRepresentation(
val actors = scenario.threads[i].map { it.toString() }.toMutableList()

if (failure is ValidationFailure) {
actors += "${failure.validationFunctionName}(): ${failure.exception::class.simpleName}"
actors += "${failure.validationFunctionName}()"
}

actors
} else scenario.threads[i].map { it.toString() }
}
}

private operator fun ExecutionResult?.get(iThread: Int, actorId: Int): Result? =
this?.threadsResults?.get(iThread)?.get(actorId)

/**
* Create a new trace node and add it to the end of the list.
*/
Expand Down Expand Up @@ -427,7 +468,11 @@ internal class ActorResultNode(
iThread: Int,
last: TraceNode?,
callDepth: Int,
internal val resultRepresentation: String?
internal val resultRepresentation: String?,
/**
* This value presents only if an exception was the actor result.
*/
internal val exceptionNumberIfExceptionResult: Int?
) : TraceNode(iThread, last, callDepth) {
override val lastState: String? = null
override val lastInternalEvent: TraceNode = this
Expand Down
60 changes: 54 additions & 6 deletions ...in/org/jetbrains/kotlinx/lincheck/strategy/managed/modelchecking/ModelCheckingStrategy.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ package org.jetbrains.kotlinx.lincheck.strategy.managed.modelchecking

import sun.nio.ch.lincheck.TestThread
import org.jetbrains.kotlinx.lincheck.*
import org.jetbrains.kotlinx.lincheck.ExceptionNumberAndStacktrace
import org.jetbrains.kotlinx.lincheck.execution.*
import org.jetbrains.kotlinx.lincheck.runner.*
import org.jetbrains.kotlinx.lincheck.strategy.*
Expand Down Expand Up @@ -86,16 +85,21 @@ internal class ModelCheckingStrategy(
*/
private fun runReplayIfPluginEnabled(failure: LincheckFailure) {
if (replay && failure.trace != null) {
// extract trace representation in the appropriate view
// Extract trace representation in the appropriate view.
val trace = constructTraceForPlugin(failure, failure.trace)
// provide all information about the failed test to the debugger
// Collect and analyze the exceptions thrown.
val (exceptionsRepresentation, internalBugOccurred) = collectExceptionsForPlugin(failure)
// If an internal bug occurred - print it on the console, no need to debug it.
if (internalBugOccurred) return
// Provide all information about the failed test to the debugger.
testFailed(
failureType = failure.type,
trace = trace,
version = lincheckVersion,
minimalPluginVersion = MINIMAL_PLUGIN_VERSION
minimalPluginVersion = MINIMAL_PLUGIN_VERSION,
exceptions = exceptionsRepresentation
)
// replay execution while it's needed
// Replay execution while it's needed.
doReplay()
while (shouldReplayInterleaving()) {
doReplay()
Expand Down Expand Up @@ -137,6 +141,47 @@ internal class ModelCheckingStrategy(
return runInvocation()
}

/**
* Processes the exceptions was thrown during the execution.
* @return exceptions string representation to pass
* to the plugin with a flag, indicating if an internal bug was the cause of the failure, or not.
*/
private fun collectExceptionsForPlugin(failure: LincheckFailure): ExceptionProcessingResult {
val results: ExecutionResult = when (failure) {
is IncorrectResultsFailure -> (failure as? IncorrectResultsFailure)?.results ?: return ExceptionProcessingResult(emptyArray(), isInternalBugOccurred = false)
is ValidationFailure -> return ExceptionProcessingResult(arrayOf(failure.exception.text), isInternalBugOccurred = false)
else -> return ExceptionProcessingResult(emptyArray(), isInternalBugOccurred = false)
}
return when (val exceptionsProcessingResult = collectExceptionStackTraces(results)) {
// If some exception was thrown from the Lincheck itself, we'll ask for bug reporting
is InternalLincheckBugResult ->
ExceptionProcessingResult(arrayOf(exceptionsProcessingResult.exception.text), isInternalBugOccurred = true)
// Otherwise collect all the exceptions
is ExceptionStackTracesResult -> {
exceptionsProcessingResult.exceptionStackTraces.entries
.sortedBy { (_, numberAndStackTrace) -> numberAndStackTrace.number }
.map { (exception, numberAndStackTrace) ->
val header = exception::class.java.canonicalName + ": " + exception.message
header + numberAndStackTrace.stackTrace.joinToString("") { "\n\tat $it" }
}
.let { ExceptionProcessingResult(it.toTypedArray(), isInternalBugOccurred = false) }
}
}
}

/**
* Result of creating string representations of exceptions
* thrown during the execution before passing them to the plugin.
*
* @param exceptionsRepresentation string representation of all the exceptions
* @param isInternalBugOccurred a flag indicating that the exception is caused by a bug in the Lincheck.
*/
@Suppress("ArrayInDataClass")
private data class ExceptionProcessingResult(
val exceptionsRepresentation: Array<String>,
val isInternalBugOccurred: Boolean
)

/**
* Transforms failure trace to the array of string to pass it to the debugger.
* (due to difficulties with passing objects like List and TracePoint, as class versions may vary)
Expand Down Expand Up @@ -209,7 +254,7 @@ internal class ModelCheckingStrategy(
is ActorResultNode -> {
val beforeEventId = -1
val representation = node.resultRepresentation.toString()
representations.add("2;${node.iThread};${node.callDepth};${node.shouldBeExpanded(false)};${beforeEventId};${representation}")
representations.add("2;${node.iThread};${node.callDepth};${node.shouldBeExpanded(false)};${beforeEventId};${representation};${node.exceptionNumberIfExceptionResult ?: -1}")
}

else -> {}
Expand All @@ -224,6 +269,9 @@ internal class ModelCheckingStrategy(
}

private fun collectExceptionsOrEmpty(failure: LincheckFailure): Map<Throwable, ExceptionNumberAndStacktrace> {
if (failure is ValidationFailure) {
return mapOf(failure.exception to ExceptionNumberAndStacktrace(1, failure.exception.stackTrace.toList()))
}
val results = (failure as? IncorrectResultsFailure)?.results ?: return emptyMap()
return when (val result = collectExceptionStackTraces(results)) {
is ExceptionStackTracesResult -> result.exceptionStackTraces
Expand Down
21 changes: 11 additions & 10 deletions src/jvm/test/resources/expected_logs/validation_function_failure.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@


java.lang.IllegalStateException: Validation works!
at org.jetbrains.kotlinx.lincheck_test.representation.ValidationFunctionCallTest.validateWithError(ValidationFunctionTests.kt:45)
at org.jetbrains.kotlinx.lincheck.runner.TestThreadExecution8218.run(Unknown Source)
at org.jetbrains.kotlinx.lincheck.runner.FixedActiveThreadsExecutor.testThreadRunnable$lambda$8(FixedActiveThreadsExecutor.kt:151)
at org.jetbrains.kotlinx.lincheck_test.representation.ValidationFunctionCallTest.validateWithError(ValidationFunctionTests.kt:44)
at org.jetbrains.kotlinx.lincheck.runner.TestThreadExecution196.run(Unknown Source)
at org.jetbrains.kotlinx.lincheck.runner.FixedActiveThreadsExecutor.testThreadRunnable$lambda$10(FixedActiveThreadsExecutor.kt:174)
at java.base/java.lang.Thread.run(Thread.java:840)


Expand All @@ -38,18 +38,19 @@ Detailed trace:
| Thread 1 | Thread 2 |
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| operation() | |
| validateInvoked.READ: 0 at ValidationFunctionCallTest.operation(ValidationFunctionTests.kt:32) | |
| validateInvoked.READ: 0 at ValidationFunctionCallTest.operation(ValidationFunctionTests.kt:36) | |
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| | operation() |
| | validateInvoked.READ: 0 at ValidationFunctionCallTest.operation(ValidationFunctionTests.kt:32) |
| | validateInvoked.READ: 0 at ValidationFunctionCallTest.operation(ValidationFunctionTests.kt:36) |
| operation() | |
| validateInvoked.READ: 0 at ValidationFunctionCallTest.operation(ValidationFunctionTests.kt:32) | |
| validateInvoked.READ: 0 at ValidationFunctionCallTest.operation(ValidationFunctionTests.kt:36) | |
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| operation() | |
| validateInvoked.READ: 0 at ValidationFunctionCallTest.operation(ValidationFunctionTests.kt:32) | |
| validateInvoked.READ: 0 at ValidationFunctionCallTest.operation(ValidationFunctionTests.kt:36) | |
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| validateWithError(): IllegalStateException | |
| validateInvoked.READ: 0 at ValidationFunctionCallTest.validateWithError(ValidationFunctionTests.kt:44) | |
| validateInvoked.WRITE(1) at ValidationFunctionCallTest.validateWithError(ValidationFunctionTests.kt:44) | |
| validateInvoked.READ: 1 at ValidationFunctionCallTest.validateWithError(ValidationFunctionTests.kt:45) | |
| validateInvoked.READ: 0 at ValidationFunctionCallTest.validateWithError(ValidationFunctionTests.kt:43) | |
| validateInvoked.WRITE(1) at ValidationFunctionCallTest.validateWithError(ValidationFunctionTests.kt:43) | |
| validateInvoked.READ: 1 at ValidationFunctionCallTest.validateWithError(ValidationFunctionTests.kt:44) | |
| result: IllegalStateException | |
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |

0 comments on commit 07e2798

Please sign in to comment.