Skip to content

Support skipping control queries in verifier #11921

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
arhimondr merged 2 commits into from
Apr 14, 2022
Merged

Support skipping control queries in verifier #11921

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
686ed21
Support skipping control runs in verifier
arhimondr Apr 12, 2022
8bc0e13
Record runtime information for failed queries in verifier
arhimondr Apr 13, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion service/trino-verifier/src/main/java/io/trino/verifier/QueryResult.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public class QueryResult
{
public enum State
{
INVALID, FAILED, SUCCESS, TOO_MANY_ROWS, TIMEOUT, FAILED_TO_SETUP, FAILED_TO_TEARDOWN
INVALID, FAILED, SUCCESS, TOO_MANY_ROWS, TIMEOUT, FAILED_TO_SETUP, FAILED_TO_TEARDOWN, SKIPPED
}

private final State state;
Expand Down
46 changes: 30 additions & 16 deletions service/trino-verifier/src/main/java/io/trino/verifier/Validator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ public class Validator
private final int controlTeardownRetries;
private final int testTeardownRetries;
private final boolean runTearDownOnResultMismatch;
private final boolean skipControl;

private Boolean valid;

Expand Down Expand Up @@ -116,6 +117,7 @@ public Validator(
int controlTeardownRetries,
int testTeardownRetries,
boolean runTearDownOnResultMismatch,
boolean skipControl,
QueryPair queryPair)
{
this.testUsername = requireNonNull(queryPair.getTest().getUsername(), "test username is null");
Expand All @@ -135,6 +137,7 @@ public Validator(
this.controlTeardownRetries = controlTeardownRetries;
this.testTeardownRetries = testTeardownRetries;
this.runTearDownOnResultMismatch = runTearDownOnResultMismatch;
this.skipControl = skipControl;

this.queryPair = requireNonNull(queryPair, "queryPair is null");
this.controlSessionProperties = queryPair.getControl().getSessionProperties();
Expand All @@ -147,7 +150,7 @@ public boolean isSkipped()
return true;
}

if (getControlResult().getState() != State.SUCCESS) {
if (!skipControl && getControlResult().getState() != State.SUCCESS) {
return true;
}

Expand Down Expand Up @@ -213,27 +216,32 @@ private boolean validate()
boolean tearDownControl = true;
boolean tearDownTest = false;
try {
controlResult = executePreAndMainForControl();
if (skipControl) {
controlResult = new QueryResult(State.SKIPPED, null, null, null, null, ImmutableList.of());
}
else {
controlResult = executePreAndMainForControl();
}

// query has too many rows. Consider banning it.
if (controlResult.getState() == State.TOO_MANY_ROWS) {
testResult = new QueryResult(State.INVALID, null, null, null, null, ImmutableList.of());
return false;
}
// query failed in the control
if (controlResult.getState() != State.SUCCESS) {
if (!skipControl && controlResult.getState() != State.SUCCESS) {
testResult = new QueryResult(State.INVALID, null, null, null, null, ImmutableList.of());
return true;
}

testResult = executePreAndMainForTest();
tearDownTest = true;

if (controlResult.getState() != State.SUCCESS || testResult.getState() != State.SUCCESS) {
if ((!skipControl && controlResult.getState() != State.SUCCESS) || testResult.getState() != State.SUCCESS) {
return false;
}

if (!checkCorrectness) {
if (skipControl || !checkCorrectness) {
return true;
}

Expand Down Expand Up @@ -481,27 +489,28 @@ private QueryResult executeQuery(String url, String username, String password, Q
ExecutorService executor = newSingleThreadExecutor();
TimeLimiter limiter = SimpleTimeLimiter.create(executor);

long start = System.nanoTime();
String queryId = null;
Duration queryCpuTime = null;
try (Connection connection = DriverManager.getConnection(url, username, password)) {
trySetConnectionProperties(query, connection);
for (Map.Entry<String, String> entry : sessionProperties.entrySet()) {
connection.unwrap(TrinoConnection.class).setSessionProperty(entry.getKey(), entry.getValue());
}

ProgressMonitor progressMonitor = new ProgressMonitor();
List<List<Object>> results;
try (Statement statement = connection.createStatement()) {
Stopwatch stopwatch = Stopwatch.createStarted();
Statement limitedStatement = limiter.newProxy(statement, Statement.class, timeout.toMillis(), MILLISECONDS);
if (explainOnly) {
sql = "EXPLAIN " + sql;
}

long start = System.nanoTime();
TrinoStatement trinoStatement = limitedStatement.unwrap(TrinoStatement.class);
ProgressMonitor progressMonitor = new ProgressMonitor();
trinoStatement.setProgressMonitor(progressMonitor);
boolean isSelectQuery = limitedStatement.execute(sql);

List<List<Object>> results;
if (isSelectQuery) {
ResultSetConverter converter = limiter.newProxy(
this::convertJdbcResultSet,
Expand All @@ -515,14 +524,19 @@ private QueryResult executeQuery(String url, String username, String password, Q
}

trinoStatement.clearProgressMonitor();
QueryStats queryStats = progressMonitor.getFinalQueryStats();
if (queryStats == null) {
if (progressMonitor.getFinalQueryStats() == null) {
throw new VerifierException("Cannot fetch query stats");
}
Duration queryCpuTime = new Duration(queryStats.getCpuTimeMillis(), MILLISECONDS);
queryId = queryStats.getQueryId();
return new QueryResult(State.SUCCESS, null, nanosSince(start), queryCpuTime, queryId, results);
}
finally {
QueryStats queryStats = progressMonitor.getFinalQueryStats();
if (queryStats != null) {
queryCpuTime = new Duration(queryStats.getCpuTimeMillis(), MILLISECONDS);
queryId = queryStats.getQueryId();
}
}

return new QueryResult(State.SUCCESS, null, nanosSince(start), queryCpuTime, queryId, results);
}
catch (SQLException e) {
Exception exception = e;
Expand All @@ -531,13 +545,13 @@ private QueryResult executeQuery(String url, String username, String password, Q
exception = (Exception) e.getCause();
}
State state = isPrestoQueryInvalid(e) ? State.INVALID : State.FAILED;
return new QueryResult(state, exception, null, null, queryId, ImmutableList.of());
return new QueryResult(state, exception, nanosSince(start), queryCpuTime, queryId, ImmutableList.of());
}
catch (VerifierException e) {
return new QueryResult(State.TOO_MANY_ROWS, e, null, null, queryId, ImmutableList.of());
return new QueryResult(State.TOO_MANY_ROWS, e, nanosSince(start), queryCpuTime, queryId, ImmutableList.of());
}
catch (UncheckedTimeoutException e) {
return new QueryResult(State.TIMEOUT, e, null, null, queryId, ImmutableList.of());
return new QueryResult(State.TIMEOUT, e, nanosSince(start), queryCpuTime, queryId, ImmutableList.of());
Comment on lines 548 to 554
Copy link
Member

@losipiuk losipiuk Apr 13, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

those look unrelated - could be separate commit.

Copy link
Contributor Author

@arhimondr arhimondr Apr 13, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a part of the Record query id for failed queries in verifier. Maybe i should change the commit message to something like Record query id and runtime information for failed queries in verifier?

}
finally {
executor.shutdownNow();
Expand Down
1 change: 1 addition & 0 deletions service/trino-verifier/src/main/java/io/trino/verifier/Verifier.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ public int run(List<QueryPair> queries)
config.getControlTeardownRetries(),
config.getTestTeardownRetries(),
config.getRunTearDownOnResultMismatch(),
config.isSkipControl(),
query);
completionService.submit(validator::valid, validator);
queriesSubmitted++;
Expand Down
13 changes: 13 additions & 0 deletions service/trino-verifier/src/main/java/io/trino/verifier/VerifierConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ public class VerifierConfig
private String shadowTestTablePrefix = "tmp_verifier_";
private String shadowControlTablePrefix = "tmp_verifier_";
private boolean runTearDownOnResultMismatch;
private boolean skipControl;

private Duration regressionMinCpuTime = new Duration(5, TimeUnit.MINUTES);

Expand Down Expand Up @@ -780,4 +781,16 @@ public VerifierConfig setRunTearDownOnResultMismatch(boolean runTearDownOnResult
this.runTearDownOnResultMismatch = runTearDownOnResultMismatch;
return this;
}

public boolean isSkipControl()
{
return skipControl;
}

@Config("skip-control")
public VerifierConfig setSkipControl(boolean skipControl)
{
this.skipControl = skipControl;
return this;
}
}
7 changes: 5 additions & 2 deletions service/trino-verifier/src/test/java/io/trino/verifier/TestVerifierConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,8 @@ public void testDefaults()
.setShadowControlTablePrefix("tmp_verifier_")
.setControlTeardownRetries(1)
.setTestTeardownRetries(1)
.setRunTearDownOnResultMismatch(false));
.setRunTearDownOnResultMismatch(false)
.setSkipControl(false));
}

@Test
Expand Down Expand Up @@ -135,6 +136,7 @@ public void testExplicitPropertyMappings()
.put("control.teardown-retries", "5")
.put("test.teardown-retries", "7")
.put("run-teardown-on-result-mismatch", "true")
.put("skip-control", "true")
.buildOrThrow();

VerifierConfig expected = new VerifierConfig().setTestUsernameOverride("verifier-test")
Expand Down Expand Up @@ -184,7 +186,8 @@ public void testExplicitPropertyMappings()
.setShadowControlTablePrefix("schema.tmp_")
.setControlTeardownRetries(5)
.setTestTeardownRetries(7)
.setRunTearDownOnResultMismatch(true);
.setRunTearDownOnResultMismatch(true)
.setSkipControl(true);

assertFullMapping(properties, expected);
}
Expand Down