Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Exceptions provided to the plugin #304

Merged
merged 10 commits into from
Apr 23, 2024
Merged

Exceptions provided to the plugin #304

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
aea516e
Exceptions provided to plugin via testFailed method
Apr 17, 2024
651b01f
cosmetics
eupp Apr 17, 2024
0b59e12
Exceptions provided to plugin via testFailed method
Apr 23, 2024
b633e0f
Merge branch 'develop' into idea-plugin-exceptions
Apr 23, 2024
d5e948c
fixes
Apr 23, 2024
a5e2ba2
fixes
Apr 23, 2024
f57e624
Validation function exception provided to the plugin
Apr 23, 2024
24b658f
Validation function exception provided to the plugin
Apr 23, 2024
1748a78
cosmetics
eupp Apr 23, 2024
d9d8aca
flatten maps in ExecutionResultsProvider
eupp Apr 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions src/jvm/main/org/jetbrains/kotlinx/lincheck/IdeaPlugin.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,18 +27,20 @@ const val MINIMAL_PLUGIN_VERSION = "0.2"
* then [beforeEvent] method is called on each trace point.
*
* @param failureType string representation of the failure type.
* (`INCORRECT_RESULTS`, `OBSTRUCTION_FREEDOM_VIOLATION`, `UNEXPECTED_EXCEPTION`, `VALIDATION_FAILURE`, `DEADLOCK`).
* (`INCORRECT_RESULTS`, `OBSTRUCTION_FREEDOM_VIOLATION`, `UNEXPECTED_EXCEPTION`, `VALIDATION_FAILURE`, `DEADLOCK` or `INTERNAL_BUG`).
* @param trace failed test trace, where each trace point is represented as a string
* (because it's the easiest way to provide some information to the debugger).
* @param version current Lincheck version
* @param minimalPluginVersion minimal compatible plugin version
* @param exceptions representation of the exceptions with their stacktrace occurred during the execution
*/
@Suppress("UNUSED_PARAMETER")
fun testFailed(
failureType: String,
trace: Array<String>,
version: String?,
minimalPluginVersion: String
minimalPluginVersion: String,
exceptions: Array<String>
) {
}

Expand Down
8 changes: 8 additions & 0 deletions src/jvm/main/org/jetbrains/kotlinx/lincheck/Utils.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ import org.jetbrains.kotlinx.lincheck.runner.*
import org.jetbrains.kotlinx.lincheck.strategy.managed.*
import org.jetbrains.kotlinx.lincheck.transformation.LincheckClassFileTransformer
import org.jetbrains.kotlinx.lincheck.verifier.*
import java.io.PrintWriter
import java.io.StringWriter
import java.lang.ref.*
import java.lang.reflect.*
import java.lang.reflect.Method
Expand Down Expand Up @@ -186,6 +188,12 @@ internal fun exceptionCanBeValidExecutionResult(exception: Throwable): Boolean {
exception !is ForcibleExecutionFinishError
}

internal val Throwable.text: String get() {
val writer = StringWriter()
printStackTrace(PrintWriter(writer))
return writer.buffer.toString()
}

/**
* Utility exception for test purposes.
* When this exception is thrown by an operation, it will halt testing with [UnexpectedExceptionInvocationResult].
Expand Down
63 changes: 54 additions & 9 deletions src/jvm/main/org/jetbrains/kotlinx/lincheck/strategy/managed/TraceReporter.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,7 @@ internal fun constructTraceGraph(
trace: Trace,
exceptionStackTraces: Map<Throwable, ExceptionNumberAndStacktrace>
): List<TraceNode> {
val resultProvider = ExecutionResultsProvider(results, failure)
val scenario = failure.scenario
val tracePoints = trace.trace
// last events that were executed for each thread. It is either thread finish events or events before crash
Expand Down Expand Up @@ -176,7 +177,7 @@ internal fun constructTraceGraph(
last = lastNode,
callDepth = 0,
actorRepresentation = actorRepresentations[iThread][nextActor],
resultRepresentation = results[iThread, nextActor]
resultRepresentation = resultProvider[iThread, nextActor]
?.let { actorNodeResultRepresentation(it, exceptionStackTraces) }
)
}
Expand Down Expand Up @@ -209,7 +210,7 @@ internal fun constructTraceGraph(
for (iThread in actorNodes.indices) {
for (actorId in actorNodes[iThread].indices) {
var actorNode = actorNodes[iThread][actorId]
val actorResult = results[iThread, actorId]
val actorResult = resultProvider[iThread, actorId]
// in case of empty trace, we want to show at least the actor nodes themselves;
// however, no actor nodes will be created by the code above, so we need to create them explicitly here.
if (actorNode == null && actorResult != null) {
Expand All @@ -229,9 +230,15 @@ internal fun constructTraceGraph(
// insert an ActorResultNode between the last actor event and the next event after it
val lastEvent = actorNode.lastInternalEvent
val lastEventNext = lastEvent.next
val result = results[iThread, actorId]
val result = resultProvider[iThread, actorId]
val resultRepresentation = result?.let { resultRepresentation(result, exceptionStackTraces) }
val resultNode = ActorResultNode(iThread, lastEvent, actorNode.callDepth + 1, resultRepresentation)
val resultNode = ActorResultNode(
iThread = iThread,
last = lastEvent,
callDepth = actorNode.callDepth + 1,
resultRepresentation = resultRepresentation,
exceptionNumberIfExceptionResult = if (result is ExceptionResult) exceptionStackTraces[result.throwable]?.number else null
)
actorNode.addInternalEvent(resultNode)
resultNode.next = lastEventNext
}
Expand All @@ -244,6 +251,43 @@ internal fun constructTraceGraph(
return traceGraphNodesSections.map { it.first() }
}

/**
* Helper class to provider execution results, including a validation function result
*/
private class ExecutionResultsProvider(result: ExecutionResult?, failure: LincheckFailure) {

/**
* A map of type Map<(threadId, actorId) -> Result>
*/
private val threadNumberToActorResultMap: Map<Pair<Int, Int>, Result> = when {
// If the results of the failure are present, then just collect them to a map.
// In that case, we know that the failure reason is not validation function, so we ignore it.
(result != null) -> {
result.threadsResults
.flatMapIndexed { tId, actors -> actors.flatMapIndexed { actorId, result ->
listOf((tId to actorId) to result)
}}
.toMap()
}

// If validation function is the reason if the failure then the only result we're interested in
// is the validation function exception.
failure is ValidationFailure -> {
mapOf((0 to firstThreadActorCount(failure)) to ExceptionResult.create(failure.exception, false))
}

else -> emptyMap()
}

operator fun get(iThread: Int, actorId: Int): Result? {
return threadNumberToActorResultMap[iThread to actorId]
}

private fun firstThreadActorCount(failure: ValidationFailure): Int =
failure.scenario.initExecution.size + failure.scenario.parallelExecution[0].size + failure.scenario.postExecution.size

}

/**
* Creates united actors representation, including invoked actors and validation functions.
* In output construction, we treat validation function call like a regular actor for unification.
Expand All @@ -257,17 +301,14 @@ private fun createActorRepresentation(
val actors = scenario.threads[i].map { it.toString() }.toMutableList()

if (failure is ValidationFailure) {
actors += "${failure.validationFunctionName}(): ${failure.exception::class.simpleName}"
actors += "${failure.validationFunctionName}()"
}

actors
} else scenario.threads[i].map { it.toString() }
}
}

private operator fun ExecutionResult?.get(iThread: Int, actorId: Int): Result? =
this?.threadsResults?.get(iThread)?.get(actorId)

/**
* Create a new trace node and add it to the end of the list.
*/
Expand Down Expand Up @@ -427,7 +468,11 @@ internal class ActorResultNode(
iThread: Int,
last: TraceNode?,
callDepth: Int,
internal val resultRepresentation: String?
internal val resultRepresentation: String?,
/**
* This value presents only if an exception was the actor result.
*/
internal val exceptionNumberIfExceptionResult: Int?
) : TraceNode(iThread, last, callDepth) {
override val lastState: String? = null
override val lastInternalEvent: TraceNode = this
Expand Down
60 changes: 54 additions & 6 deletions ...in/org/jetbrains/kotlinx/lincheck/strategy/managed/modelchecking/ModelCheckingStrategy.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ package org.jetbrains.kotlinx.lincheck.strategy.managed.modelchecking

import sun.nio.ch.lincheck.TestThread
import org.jetbrains.kotlinx.lincheck.*
import org.jetbrains.kotlinx.lincheck.ExceptionNumberAndStacktrace
import org.jetbrains.kotlinx.lincheck.execution.*
import org.jetbrains.kotlinx.lincheck.runner.*
import org.jetbrains.kotlinx.lincheck.strategy.*
Expand Down Expand Up @@ -86,16 +85,21 @@ internal class ModelCheckingStrategy(
*/
private fun runReplayIfPluginEnabled(failure: LincheckFailure) {
if (replay && failure.trace != null) {
// extract trace representation in the appropriate view
// Extract trace representation in the appropriate view.
val trace = constructTraceForPlugin(failure, failure.trace)
// provide all information about the failed test to the debugger
// Collect and analyze the exceptions thrown.
val (exceptionsRepresentation, internalBugOccurred) = collectExceptionsForPlugin(failure)
// If an internal bug occurred - print it on the console, no need to debug it.
if (internalBugOccurred) return
// Provide all information about the failed test to the debugger.
testFailed(
failureType = failure.type,
trace = trace,
version = lincheckVersion,
minimalPluginVersion = MINIMAL_PLUGIN_VERSION
minimalPluginVersion = MINIMAL_PLUGIN_VERSION,
exceptions = exceptionsRepresentation
)
// replay execution while it's needed
// Replay execution while it's needed.
doReplay()
while (shouldReplayInterleaving()) {
doReplay()
Expand Down Expand Up @@ -137,6 +141,47 @@ internal class ModelCheckingStrategy(
return runInvocation()
}

/**
* Processes the exceptions was thrown during the execution.
* @return exceptions string representation to pass
* to the plugin with a flag, indicating if an internal bug was the cause of the failure, or not.
*/
private fun collectExceptionsForPlugin(failure: LincheckFailure): ExceptionProcessingResult {
val results: ExecutionResult = when (failure) {
is IncorrectResultsFailure -> (failure as? IncorrectResultsFailure)?.results ?: return ExceptionProcessingResult(emptyArray(), isInternalBugOccurred = false)
is ValidationFailure -> return ExceptionProcessingResult(arrayOf(failure.exception.text), isInternalBugOccurred = false)
else -> return ExceptionProcessingResult(emptyArray(), isInternalBugOccurred = false)
}
return when (val exceptionsProcessingResult = collectExceptionStackTraces(results)) {
// If some exception was thrown from the Lincheck itself, we'll ask for bug reporting
is InternalLincheckBugResult ->
ExceptionProcessingResult(arrayOf(exceptionsProcessingResult.exception.text), isInternalBugOccurred = true)
// Otherwise collect all the exceptions
is ExceptionStackTracesResult -> {
exceptionsProcessingResult.exceptionStackTraces.entries
.sortedBy { (_, numberAndStackTrace) -> numberAndStackTrace.number }
.map { (exception, numberAndStackTrace) ->
val header = exception::class.java.canonicalName + ": " + exception.message
header + numberAndStackTrace.stackTrace.joinToString("") { "\n\tat $it" }
}
.let { ExceptionProcessingResult(it.toTypedArray(), isInternalBugOccurred = false) }
}
}
}

/**
* Result of creating string representations of exceptions
* thrown during the execution before passing them to the plugin.
*
* @param exceptionsRepresentation string representation of all the exceptions
* @param isInternalBugOccurred a flag indicating that the exception is caused by a bug in the Lincheck.
*/
@Suppress("ArrayInDataClass")
private data class ExceptionProcessingResult(
val exceptionsRepresentation: Array<String>,
val isInternalBugOccurred: Boolean
)

/**
* Transforms failure trace to the array of string to pass it to the debugger.
* (due to difficulties with passing objects like List and TracePoint, as class versions may vary)
Expand Down Expand Up @@ -209,7 +254,7 @@ internal class ModelCheckingStrategy(
is ActorResultNode -> {
val beforeEventId = -1
val representation = node.resultRepresentation.toString()
representations.add("2;${node.iThread};${node.callDepth};${node.shouldBeExpanded(false)};${beforeEventId};${representation}")
representations.add("2;${node.iThread};${node.callDepth};${node.shouldBeExpanded(false)};${beforeEventId};${representation};${node.exceptionNumberIfExceptionResult ?: -1}")
}

else -> {}
Expand All @@ -224,6 +269,9 @@ internal class ModelCheckingStrategy(
}

private fun collectExceptionsOrEmpty(failure: LincheckFailure): Map<Throwable, ExceptionNumberAndStacktrace> {
if (failure is ValidationFailure) {
return mapOf(failure.exception to ExceptionNumberAndStacktrace(1, failure.exception.stackTrace.toList()))
}
val results = (failure as? IncorrectResultsFailure)?.results ?: return emptyMap()
return when (val result = collectExceptionStackTraces(results)) {
is ExceptionStackTracesResult -> result.exceptionStackTraces
Expand Down
21 changes: 11 additions & 10 deletions src/jvm/test/resources/expected_logs/validation_function_failure.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@


java.lang.IllegalStateException: Validation works!
at org.jetbrains.kotlinx.lincheck_test.representation.ValidationFunctionCallTest.validateWithError(ValidationFunctionTests.kt:45)
at org.jetbrains.kotlinx.lincheck.runner.TestThreadExecution8218.run(Unknown Source)
at org.jetbrains.kotlinx.lincheck.runner.FixedActiveThreadsExecutor.testThreadRunnable$lambda$8(FixedActiveThreadsExecutor.kt:151)
at org.jetbrains.kotlinx.lincheck_test.representation.ValidationFunctionCallTest.validateWithError(ValidationFunctionTests.kt:44)
at org.jetbrains.kotlinx.lincheck.runner.TestThreadExecution196.run(Unknown Source)
at org.jetbrains.kotlinx.lincheck.runner.FixedActiveThreadsExecutor.testThreadRunnable$lambda$10(FixedActiveThreadsExecutor.kt:174)
at java.base/java.lang.Thread.run(Thread.java:840)


Expand All @@ -38,18 +38,19 @@ Detailed trace:
| Thread 1 | Thread 2 |
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| operation() | |
| validateInvoked.READ: 0 at ValidationFunctionCallTest.operation(ValidationFunctionTests.kt:32) | |
| validateInvoked.READ: 0 at ValidationFunctionCallTest.operation(ValidationFunctionTests.kt:36) | |
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| | operation() |
| | validateInvoked.READ: 0 at ValidationFunctionCallTest.operation(ValidationFunctionTests.kt:32) |
| | validateInvoked.READ: 0 at ValidationFunctionCallTest.operation(ValidationFunctionTests.kt:36) |
| operation() | |
| validateInvoked.READ: 0 at ValidationFunctionCallTest.operation(ValidationFunctionTests.kt:32) | |
| validateInvoked.READ: 0 at ValidationFunctionCallTest.operation(ValidationFunctionTests.kt:36) | |
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| operation() | |
| validateInvoked.READ: 0 at ValidationFunctionCallTest.operation(ValidationFunctionTests.kt:32) | |
| validateInvoked.READ: 0 at ValidationFunctionCallTest.operation(ValidationFunctionTests.kt:36) | |
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| validateWithError(): IllegalStateException | |
| validateInvoked.READ: 0 at ValidationFunctionCallTest.validateWithError(ValidationFunctionTests.kt:44) | |
| validateInvoked.WRITE(1) at ValidationFunctionCallTest.validateWithError(ValidationFunctionTests.kt:44) | |
| validateInvoked.READ: 1 at ValidationFunctionCallTest.validateWithError(ValidationFunctionTests.kt:45) | |
| validateInvoked.READ: 0 at ValidationFunctionCallTest.validateWithError(ValidationFunctionTests.kt:43) | |
| validateInvoked.WRITE(1) at ValidationFunctionCallTest.validateWithError(ValidationFunctionTests.kt:43) | |
| validateInvoked.READ: 1 at ValidationFunctionCallTest.validateWithError(ValidationFunctionTests.kt:44) | |
| result: IllegalStateException | |
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |