Skip to content

Commit

Permalink
Minor fixes to QueryStage
Browse files Browse the repository at this point in the history
- Add javadoc to QueryStep
- Rename QueryStep.kind to name
- Make QueryStep.substeps an empty list if not set
- Document that QueryStatistics.queryPlan can return null
- Add check for query plan in ITBigQueryTest.testQueryJob
- Add unit tests for QueryStep
- Better javadoc for QueryPlan getters
  • Loading branch information
mziccard committed Jan 6, 2016
1 parent 4aa8a97 commit 13b1b0e
Show file tree
Hide file tree
Showing 4 changed files with 52 additions and 32 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -339,10 +339,10 @@ public Long totalBytesProcessed() {
}

/**
* Returns the query plan as a list of stages. Each stage involves a number of steps that read
* from data sources, perform a series of transformations on the input, and emit an output to a
* future stage (or the final result). The query plan is available for a completed query job and
* is retained for 7 days.
* Returns the query plan as a list of stages or {@code null} if a query plan is not available.
* Each stage involves a number of steps that read from data sources, perform a series of
* transformations on the input, and emit an output to a future stage (or the final result). The
* query plan is available for a completed query job and is retained for 7 days.
*
* @see <a href="https://cloud.google.com/bigquery/query-plan-explanation">Query Plan</a>
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,33 +78,42 @@ public ExplainQueryStep apply(QueryStep stage) {
};
private static final long serialVersionUID = 8663444604771794411L;

private final String kind;
private final String name;
private final List<String> substeps;

QueryStep(String kind, List<String> substeps) {
this.kind = kind;
QueryStep(String name, List<String> substeps) {
this.name = name;
this.substeps = substeps;
}

public String kind() {
return kind;
/**
* Returns a machine-readable name for the operation.
*
* @see <a href="https://cloud.google.com/bigquery/query-plan-explanation#steps_metadata">Steps
* Metadata</a>
*/
public String name() {
return name;
}

/**
* Returns a list of human-readable stage descriptions.
*/
public List<String> substeps() {
return substeps;
}

@Override
public String toString() {
return MoreObjects.toStringHelper(this)
.add("kind", kind)
.add("name", name)
.add("substeps", substeps)
.toString();
}

@Override
public int hashCode() {
return Objects.hash(kind, substeps);
return Objects.hash(name, substeps);
}

@Override
Expand All @@ -113,20 +122,16 @@ public boolean equals(Object obj) {
return false;
}
QueryStep other = (QueryStep) obj;
return Objects.equals(kind, other.kind)
&& Objects.equals(substeps, other.substeps);
return Objects.equals(name, other.name) && Objects.equals(substeps, other.substeps);
}

ExplainQueryStep toPb() {
return new ExplainQueryStep().setKind(kind).setSubsteps(substeps);
return new ExplainQueryStep().setKind(name).setSubsteps(substeps);
}

static QueryStep fromPb(com.google.api.services.bigquery.model.ExplainQueryStep stepPb) {
List<String> substeps = null;
if (stepPb.getSubsteps() != null) {
substeps = ImmutableList.copyOf(stepPb.getSubsteps());
}
return new QueryStep(stepPb.getKind(), substeps);
return new QueryStep(stepPb.getKind(), ImmutableList.copyOf(stepPb.getSubsteps() != null
? stepPb.getSubsteps() : ImmutableList.<String>of()));
}
}

Expand Down Expand Up @@ -249,14 +254,16 @@ QueryStage build() {
}

/**
* Returns the relative amount of time the average shard spent on CPU-bound tasks.
* Returns the time the average worker spent CPU-bound, divided by the longest time spent by any
* worker in any segment.
*/
public double computeRatioAvg() {
return computeRatioAvg;
}

/**
* Returns the relative amount of time the slowest shard spent on CPU-bound tasks.
* Returns the time the slowest worker spent CPU-bound, divided by the longest time spent by any
* worker in any segment.
*/
public double computeRatioMax() {
return computeRatioMax;
Expand All @@ -277,28 +284,30 @@ public String name() {
}

/**
* Returns the relative amount of time the average shard spent reading input.
* Returns the time the average worker spent reading input data, divided by the longest time spent
* by any worker in any segment.
*/
public double readRatioAvg() {
return readRatioAvg;
}

/**
* Returns the relative amount of time the slowest shard spent reading input.
* Returns the time the slowest worker spent reading input data, divided by the longest time spent
* by any worker in any segment.
*/
public double readRatioMax() {
return readRatioMax;
}

/**
* Returns the number of records read into the stage.
* Returns the number of rows (top-level records) read by the stage.
*/
public long recordsRead() {
return recordsRead;
}

/**
* Returns the number of records written by the stage.
* Returns the number of rows (top-level records) written by the stage.
*/
public long recordsWritten() {
return recordsWritten;
Expand All @@ -312,28 +321,32 @@ public List<QueryStep> steps() {
}

/**
* Returns the relative amount of time the average shard spent waiting to be scheduled.
* Returns the time the average worker spent waiting to be scheduled, divided by the longest time
* spent by any worker in any segment.
*/
public double waitRatioAvg() {
return waitRatioAvg;
}

/**
* Returns the relative amount of time the slowest shard spent waiting to be scheduled.
* Returns the time the slowest worker spent waiting to be scheduled, divided by the longest time
* spent by any worker in any segment.
*/
public double waitRatioMax() {
return waitRatioMax;
}

/**
* Returns the relative amount of time the average shard spent on writing output.
* Returns the time the average worker spent writing output data, divided by the longest time
* spent by any worker in any segment.
*/
public double writeRatioAvg() {
return writeRatioAvg;
}

/**
* Returns the relative amount of time the slowest shard spent on writing output.
* Returns the time the slowest worker spent writing output data, divided by the longest time
* spent by any worker in any segment.
*/
public double writeRatioMax() {
return writeRatioMax;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -771,6 +771,8 @@ public void testQueryJob() throws InterruptedException {
}
assertEquals(2, rowCount);
assertTrue(bigquery.delete(DATASET, tableName));
QueryJobInfo queryJob = bigquery.getJob(remoteJob.jobId());
assertNotNull(queryJob.statistics().queryPlan());
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import static org.junit.Assert.assertEquals;

import com.google.api.services.bigquery.model.ExplainQueryStep;
import com.google.common.collect.ImmutableList;
import com.google.gcloud.bigquery.QueryStage.QueryStep;

Expand Down Expand Up @@ -62,8 +63,8 @@ public class QueryStageTest {

@Test
public void testQueryStepConstructor() {
assertEquals("KIND", QUERY_STEP1.kind());
assertEquals("KIND", QUERY_STEP2.kind());
assertEquals("KIND", QUERY_STEP1.name());
assertEquals("KIND", QUERY_STEP2.name());
assertEquals(SUBSTEPS1, QUERY_STEP1.substeps());
assertEquals(SUBSTEPS2, QUERY_STEP2.substeps());
}
Expand All @@ -90,6 +91,10 @@ public void testToAndFromPb() {
compareQueryStep(QUERY_STEP1, QueryStep.fromPb(QUERY_STEP1.toPb()));
compareQueryStep(QUERY_STEP2, QueryStep.fromPb(QUERY_STEP2.toPb()));
compareQueryStage(QUERY_STAGE, QueryStage.fromPb(QUERY_STAGE.toPb()));
ExplainQueryStep stepPb = new ExplainQueryStep();
stepPb.setKind("KIND");
stepPb.setSubsteps(null);
compareQueryStep(new QueryStep("KIND", ImmutableList.<String>of()), QueryStep.fromPb(stepPb));
}

@Test
Expand Down Expand Up @@ -119,7 +124,7 @@ private void compareQueryStage(QueryStage expected, QueryStage value) {

private void compareQueryStep(QueryStep expected, QueryStep value) {
assertEquals(expected, value);
assertEquals(expected.kind(), value.kind());
assertEquals(expected.name(), value.name());
assertEquals(expected.substeps(), value.substeps());
assertEquals(expected.hashCode(), value.hashCode());
}
Expand Down

0 comments on commit 13b1b0e

Please sign in to comment.