diff --git a/src/BigQuery/BigQueryClient.php b/src/BigQuery/BigQueryClient.php index d1536fec16b2..f0e1f0772bf5 100644 --- a/src/BigQuery/BigQueryClient.php +++ b/src/BigQuery/BigQueryClient.php @@ -19,11 +19,14 @@ use Google\Cloud\BigQuery\Connection\ConnectionInterface; use Google\Cloud\BigQuery\Connection\Rest; +use Google\Cloud\BigQuery\Exception\JobException; +use Google\Cloud\BigQuery\Job; use Google\Cloud\Core\ArrayTrait; -use Google\Cloud\Core\Iterator\ItemIterator; -use Google\Cloud\Core\Iterator\PageIterator; use Google\Cloud\Core\ClientTrait; use Google\Cloud\Core\Int64; +use Google\Cloud\Core\Iterator\ItemIterator; +use Google\Cloud\Core\Iterator\PageIterator; +use Google\Cloud\Core\RetryDeciderTrait; use Psr\Cache\CacheItemPoolInterface; use Psr\Http\Message\StreamInterface; @@ -43,10 +46,12 @@ class BigQueryClient { use ArrayTrait; use ClientTrait; - use JobConfigurationTrait; + use RetryDeciderTrait; const VERSION = '0.2.4'; + const MAX_DELAY_MICROSECONDS = 32000000; + const SCOPE = 'https://www.googleapis.com/auth/bigquery'; const INSERT_SCOPE = 'https://www.googleapis.com/auth/bigquery.insertdata'; @@ -56,7 +61,7 @@ class BigQueryClient protected $connection; /** - * @var ValueMapper $mapper Maps values between PHP and BigQuery. + * @var ValueMapper Maps values between PHP and BigQuery. */ private $mapper; @@ -95,15 +100,85 @@ class BigQueryClient */ public function __construct(array $config = []) { + $this->setHttpRetryCodes([]); + $this->setHttpRetryMessages([ + 'rateLimitExceeded', + 'backendError' + ]); $config += [ 'scopes' => [self::SCOPE], - 'returnInt64AsObject' => false + 'returnInt64AsObject' => false, + 'restRetryFunction' => $this->getRetryFunction(), + 'restDelayFunction' => function ($attempt) { + return min( + mt_rand(0, 1000000) + (pow(2, $attempt) * 1000000), + self::MAX_DELAY_MICROSECONDS + ); + } ]; $this->connection = new Rest($this->configureAuthentication($config)); $this->mapper = new ValueMapper($config['returnInt64AsObject']); } + /** + * Returns a job configuration to be passed to either + * {@see Google\Cloud\BigQuery\BigQueryClient::runQuery()} or + * {@see Google\Cloud\BigQuery\BigQueryClient::startQuery()}. A + * configuration can be built using fluent setters or by providing a full + * set of options at once. + * + * Unless otherwise specified, all configuration options will default based + * on the [Jobs configuration API documentation] + * (https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration) + * except for `configuration.query.useLegacySql`, which defaults to `false` + * in this client. + * + * Example: + * ``` + * $queryJobConfig = $bigQuery->query( + * 'SELECT commit FROM `bigquery-public-data.github_repos.commits` LIMIT 100' + * ); + * ``` + * + * ``` + * // Set create disposition using fluent setters. + * $queryJobConfig = $bigQuery->query( + * 'SELECT commit FROM `bigquery-public-data.github_repos.commits` LIMIT 100' + * )->createDisposition('CREATE_NEVER'); + * ``` + * + * ``` + * // This is equivalent to the above example, using array configuration + * // instead of fluent setters. + * $queryJobConfig = $bigQuery->query( + * 'SELECT commit FROM `bigquery-public-data.github_repos.commits` LIMIT 100', + * [ + * 'configuration' => [ + * 'query' => [ + * 'createDisposition' => 'CREATE_NEVER' + * ] + * ] + * ] + * ); + * ``` + * + * @param string $query A BigQuery SQL query. + * @param array $options [optional] Please see the + * [API documentation for Job configuration] + * (https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration) + * for the available options. + * @return QueryJobConfiguration + */ + public function query($query, array $options = []) + { + return (new QueryJobConfiguration( + $this->mapper, + $this->projectId, + $options + ))->query($query); + } + /** * Runs a BigQuery SQL query in a synchronous fashion. Rows are returned * immediately as long as the query completes within a specified timeout. In @@ -112,7 +187,7 @@ public function __construct(array $config = []) * * Queries constructed using * [standard SQL](https://cloud.google.com/bigquery/docs/reference/standard-sql/) - * can take advantage of parametriziation. + * can take advantage of parameterization. * * Refer to the table below for a guide on how parameter types are mapped to * their BigQuery equivalents. @@ -136,17 +211,12 @@ public function __construct(array $config = []) * * Example: * ``` - * $queryResults = $bigQuery->runQuery('SELECT commit FROM [bigquery-public-data:github_repos.commits] LIMIT 100'); - * - * $isComplete = $queryResults->isComplete(); - * - * while (!$isComplete) { - * sleep(1); // let's wait for a moment... - * $queryResults->reload(); // trigger a network request - * $isComplete = $queryResults->isComplete(); // check the query's status - * } + * $queryJobConfig = $bigQuery->query( + * 'SELECT commit FROM `bigquery-public-data.github_repos.commits` LIMIT 100' + * ); + * $queryResults = $bigQuery->runQuery($queryJobConfig); * - * foreach ($queryResults->rows() as $row) { + * foreach ($queryResults as $row) { * echo $row['commit']; * } * ``` @@ -155,22 +225,14 @@ public function __construct(array $config = []) * // Construct a query utilizing named parameters. * $query = 'SELECT commit FROM `bigquery-public-data.github_repos.commits`' . * 'WHERE author.date < @date AND message = @message LIMIT 100'; - * $queryResults = $bigQuery->runQuery($query, [ - * 'parameters' => [ + * $queryJobConfig = $bigQuery->query($query) + * ->parameters([ * 'date' => $bigQuery->timestamp(new \DateTime('1980-01-01 12:15:00Z')), * 'message' => 'A commit message.' - * ] - * ]); - * - * $isComplete = $queryResults->isComplete(); - * - * while (!$isComplete) { - * sleep(1); // let's wait for a moment... - * $queryResults->reload(); // trigger a network request - * $isComplete = $queryResults->isComplete(); // check the query's status - * } + * ]); + * $queryResults = $bigQuery->runQuery($queryJobConfig); * - * foreach ($queryResults->rows() as $row) { + * foreach ($queryResults as $row) { * echo $row['commit']; * } * ``` @@ -178,26 +240,18 @@ public function __construct(array $config = []) * ``` * // Construct a query utilizing positional parameters. * $query = 'SELECT commit FROM `bigquery-public-data.github_repos.commits` WHERE message = ? LIMIT 100'; - * $queryResults = $bigQuery->runQuery($query, [ - * 'parameters' => ['A commit message.'] - * ]); - * - * $isComplete = $queryResults->isComplete(); + * $queryJobConfig = $bigQuery->query($query) + * ->parameters(['A commit message.']); + * $queryResults = $bigQuery->runQuery($queryJobConfig); * - * while (!$isComplete) { - * sleep(1); // let's wait for a moment... - * $queryResults->reload(); // trigger a network request - * $isComplete = $queryResults->isComplete(); // check the query's status - * } - * - * foreach ($queryResults->rows() as $row) { + * foreach ($queryResults as $row) { * echo $row['commit']; * } * ``` * * @see https://cloud.google.com/bigquery/docs/reference/v2/jobs/query Query API documentation. * - * @param string $query A BigQuery SQL query. + * @param QueryJobConfiguration $query A BigQuery SQL query configuration. * @param array $options [optional] { * Configuration options. * @@ -205,114 +259,66 @@ public function __construct(array $config = []) * of results. Setting this flag to a small value such as 1000 and * then paging through results might improve reliability when the * query result set is large. - * @type array $defaultDataset Specifies the default datasetId and - * projectId to assume for any unqualified table names in the - * query. If not set, all table names in the query string must be - * qualified in the format 'datasetId.tableId'. + * @type int $startIndex Zero-based index of the starting row. * @type int $timeoutMs How long to wait for the query to complete, in * milliseconds. **Defaults to** `10000` milliseconds (10 seconds). - * @type bool $useQueryCache Whether to look for the result in the query - * cache. - * @type bool $useLegacySql Specifies whether to use BigQuery's legacy - * SQL dialect for this query. **Defaults to** `true`. If set to - * false, the query will use - * [BigQuery's standard SQL](https://cloud.google.com/bigquery/sql-reference). - * @type array $parameters Only available for standard SQL queries. - * When providing a non-associative array positional parameters - * (`?`) will be used. When providing an associative array - * named parameters will be used (`@name`). + * @type int $maxRetries The number of times to retry, checking if the + * query has completed. **Defaults to** `100`. * } * @return QueryResults + * @throws JobException If the maximum number of retries while waiting for + * query completion has been exceeded. */ - public function runQuery($query, array $options = []) + public function runQuery(JobConfigurationInterface $query, array $options = []) { - if (isset($options['parameters'])) { - $options += $this->formatQueryParameters($options['parameters']); - unset($options['parameters']); - } - - $response = $this->connection->query([ - 'projectId' => $this->projectId, - 'query' => $query - ] + $options); - - return new QueryResults( - $this->connection, - $response['jobReference']['jobId'], - $this->projectId, - $response, - $options, - $this->mapper - ); + $queryResultsOptions = $this->pluckArray([ + 'maxResults', + 'startIndex', + 'timeoutMs', + 'maxRetries' + ], $options); + + return $this->startQuery( + $query, + $options + )->queryResults($queryResultsOptions + $options); } /** - * Runs a BigQuery SQL query in an asynchronous fashion. Running a query - * in this fashion requires you to poll for the status before being able - * to access results. + * Runs a BigQuery SQL query in an asynchronous fashion. * * Queries constructed using * [standard SQL](https://cloud.google.com/bigquery/docs/reference/standard-sql/) - * can take advantage of parametriziation. For more details and examples + * can take advantage of parameterization. For more details and examples * please see {@see Google\Cloud\BigQuery\BigQueryClient::runQuery()}. * * Example: * ``` - * $job = $bigQuery->runQueryAsJob('SELECT commit FROM [bigquery-public-data:github_repos.commits] LIMIT 100'); - * - * $isComplete = false; + * $queryJobConfig = $bigQuery->query( + * 'SELECT commit FROM `bigquery-public-data.github_repos.commits` LIMIT 100' + * ); + * $job = $bigQuery->startQuery($queryJobConfig); * $queryResults = $job->queryResults(); * - * while (!$isComplete) { - * sleep(1); // let's wait for a moment... - * $queryResults->reload(); // trigger a network request - * $isComplete = $queryResults->isComplete(); // check the query's status - * } - * - * foreach ($queryResults->rows() as $row) { + * foreach ($queryResults as $row) { * echo $row['commit']; * } * ``` * * @see https://cloud.google.com/bigquery/docs/reference/v2/jobs/insert Jobs insert API documentation. * - * @param string $query A BigQuery SQL query. - * @param array $options [optional] { - * Configuration options. - * - * @type array $parameters Only available for standard SQL queries. - * When providing a non-associative array positional parameters - * (`?`) will be used. When providing an associative array - * named parameters will be used (`@name`). - * @type array $jobConfig Configuration settings for a query job are - * outlined in the [API Docs for `configuration.query`](https://goo.gl/PuRa3I). - * If not provided default settings will be used. - * } + * @param QueryJobConfiguration $query A BigQuery SQL query configuration. + * @param array $options [optional] Configuration options. * @return Job */ - public function runQueryAsJob($query, array $options = []) + public function startQuery(JobConfigurationInterface $query, array $options = []) { - if (isset($options['parameters'])) { - if (!isset($options['jobConfig'])) { - $options['jobConfig'] = []; - } - - $options['jobConfig'] += $this->formatQueryParameters($options['parameters']); - unset($options['parameters']); - } - - $config = $this->buildJobConfig( - 'query', - $this->projectId, - ['query' => $query], - $options - ); - - $response = $this->connection->insertJob($config); + $config = $query->toArray(); + $response = $this->connection->insertJob($config + $options); return new Job( $this->connection, - $response['jobReference']['jobId'], + $config['jobReference']['jobId'], $this->projectId, $this->mapper, $response @@ -329,7 +335,7 @@ public function runQueryAsJob($query, array $options = []) * $job = $bigQuery->job('myJobId'); * ``` * - * @param string $id The id of the job to request. + * @param string $id The id of the already run or running job to request. * @return Job */ public function job($id) @@ -472,6 +478,9 @@ function (array $dataset) { /** * Creates a dataset. * + * Please note that by default the library will not attempt to retry this + * call on your behalf. + * * Example: * ``` * $dataset = $bigQuery->createDataset('aDataset'); @@ -496,12 +505,16 @@ public function createDataset($id, array $options = []) unset($options['metadata']); } - $response = $this->connection->insertDataset([ - 'projectId' => $this->projectId, - 'datasetReference' => [ - 'datasetId' => $id + $response = $this->connection->insertDataset( + [ + 'projectId' => $this->projectId, + 'datasetReference' => [ + 'datasetId' => $id + ] ] - ] + $options); + + $options + + ['retries' => 0] + ); return new Dataset( $this->connection, @@ -592,30 +605,4 @@ public function timestamp(\DateTimeInterface $value) { return new Timestamp($value); } - - /** - * Formats query parameters for the API. - * - * @param array $parameters The parameters to format. - * @return array - */ - private function formatQueryParameters(array $parameters) - { - $options = [ - 'parameterMode' => $this->isAssoc($parameters) ? 'named' : 'positional', - 'useLegacySql' => false - ]; - - foreach ($parameters as $name => $value) { - $param = $this->mapper->toParameter($value); - - if ($options['parameterMode'] === 'named') { - $param += ['name' => $name]; - } - - $options['queryParameters'][] = $param; - } - - return $options; - } } diff --git a/src/BigQuery/Connection/Rest.php b/src/BigQuery/Connection/Rest.php index 1118a56c0a59..c851e7553ecf 100644 --- a/src/BigQuery/Connection/Rest.php +++ b/src/BigQuery/Connection/Rest.php @@ -246,12 +246,19 @@ private function resolveUploadOptions(array $args) $args += [ 'projectId' => null, 'data' => null, - 'configuration' => [] + 'configuration' => [], + 'labels' => [], + 'dryRun' => false, + 'jobReference' => [] ]; $args['data'] = Psr7\stream_for($args['data']); - $args['metadata']['configuration'] = $args['configuration']; - unset($args['configuration']); + $args['metadata'] = $this->pluckArray([ + 'labels', + 'dryRun', + 'jobReference', + 'configuration' + ], $args); $uploaderOptionKeys = [ 'restOptions', diff --git a/src/BigQuery/CopyJobConfiguration.php b/src/BigQuery/CopyJobConfiguration.php new file mode 100644 index 000000000000..29d92aabd270 --- /dev/null +++ b/src/BigQuery/CopyJobConfiguration.php @@ -0,0 +1,154 @@ +dataset('my_dataset') + * ->table('my_source_table'); + * $destinationTable = $bigQuery->dataset('my_dataset') + * ->table('my_destination_table'); + * + * $copyJobConfig = $sourceTable->copy($destinationTable); + * ``` + */ +class CopyJobConfiguration implements JobConfigurationInterface +{ + use JobConfigurationTrait; + + /** + * @param string $projectId The project's ID. + * @param array $config A set of configuration options for a job. + */ + public function __construct($projectId, array $config) + { + $this->jobConfigurationProperties($projectId, $config); + } + + /** + * Set whether the job is allowed to create new tables. Creation, truncation + * and append actions occur as one atomic update upon job completion. + * + * Example: + * ``` + * $copyJobConfig->createDisposition('CREATE_NEVER'); + * ``` + * + * @param string $createDisposition The create disposition. Acceptable + * values include `"CREATED_IF_NEEDED"`, `"CREATE_NEVER"`. **Defaults + * to** `"CREATE_IF_NEEDED"`. + * @return CopyJobConfiguration + */ + public function createDisposition($createDisposition) + { + $this->config['configuration']['copy']['createDisposition'] = $createDisposition; + + return $this; + } + + /** + * Sets the custom encryption configuration (e.g., Cloud KMS keys). + * + * Example: + * ``` + * $copyJobConfig->destinationEncryptionConfiguration([ + * 'kmsKeyName' => 'my_key' + * ]); + * ``` + * + * @param array $configuration Custom encryption configuration. + * @return CopyJobConfiguration + */ + public function destinationEncryptionConfiguration(array $configuration) + { + $this->config['configuration']['copy']['destinationEncryptionConfiguration'] = $configuration; + + return $this; + } + + /** + * Sets the destination table. + * + * Example: + * ``` + * $table = $bigQuery->dataset('my_dataset') + * ->table('my_table'); + * $copyJobConfig->destinationTable($table); + * ``` + * + * @param Table $destinationTable The destination table. + * @return CopyJobConfiguration + */ + public function destinationTable(Table $destinationTable) + { + $this->config['configuration']['copy']['destinationTable'] = $destinationTable->identity(); + + return $this; + } + + /** + * Sets the source table to copy. + * + * Example: + * ``` + * $table = $bigQuery->dataset('my_dataset') + * ->table('source_table'); + * $copyJobConfig->sourceTable($table); + * ``` + * + * @param Table $sourceTable The destination table. + * @return CopyJobConfiguration + */ + public function sourceTable(Table $sourceTable) + { + $this->config['configuration']['copy']['sourceTable'] = $sourceTable->identity(); + + return $this; + } + + /** + * Sets the action that occurs if the destination table already exists. Each + * action is atomic and only occurs if BigQuery is able to complete the job + * successfully. Creation, truncation and append actions occur as one atomic + * update upon job completion. + * + * Example: + * ``` + * $copyJobConfig->writeDisposition('WRITE_TRUNCATE'); + * ``` + * + * @param string $writeDisposition The write disposition. Acceptable values + * include `"WRITE_TRUNCATE"`, `"WRITE_APPEND"`, `"WRITE_EMPTY"`. + * **Defaults to** `"WRITE_EMPTY"`. + * @return CopyJobConfiguration + */ + public function writeDisposition($writeDisposition) + { + $this->config['configuration']['copy']['writeDisposition'] = $writeDisposition; + + return $this; + } +} diff --git a/src/BigQuery/Dataset.php b/src/BigQuery/Dataset.php index 87d217650a1a..72defc18d874 100644 --- a/src/BigQuery/Dataset.php +++ b/src/BigQuery/Dataset.php @@ -19,6 +19,7 @@ use Google\Cloud\BigQuery\Connection\ConnectionInterface; use Google\Cloud\Core\ArrayTrait; +use Google\Cloud\Core\ConcurrencyControlTrait; use Google\Cloud\Core\Exception\NotFoundException; use Google\Cloud\Core\Iterator\ItemIterator; use Google\Cloud\Core\Iterator\PageIterator; @@ -30,6 +31,7 @@ class Dataset { use ArrayTrait; + use ConcurrencyControlTrait; /** * @var ConnectionInterface Represents a connection to BigQuery. @@ -98,6 +100,9 @@ public function exists() /** * Delete the dataset. * + * Please note that by default the library will not attempt to retry this + * call on your behalf. + * * Example: * ``` * $dataset->delete(); @@ -115,12 +120,24 @@ public function exists() */ public function delete(array $options = []) { - $this->connection->deleteDataset($options + $this->identity); + $this->connection->deleteDataset( + $options + + ['retries' => 0] + + $this->identity + ); } /** * Update the dataset. * + * Providing an `etag` key as part of `$metadata` will enable simultaneous + * update protection. This is useful in preventing override of modifications + * made by another user. The resource's current etag can be obtained via a + * GET request on the resource. + * + * Please note that by default this call will not automatically retry on + * your behalf unless an `etag` is set. + * * Example: * ``` * $dataset->update([ @@ -129,6 +146,7 @@ public function delete(array $options = []) * ``` * * @see https://cloud.google.com/bigquery/docs/reference/v2/datasets/patch Datasets patch API documentation. + * @see https://cloud.google.com/bigquery/docs/api-performance#patch Patch (Partial Update) * * @param array $metadata The available options for metadata are outlined * at the [Dataset Resource API docs](https://cloud.google.com/bigquery/docs/reference/v2/datasets#resource) @@ -136,10 +154,17 @@ public function delete(array $options = []) */ public function update(array $metadata, array $options = []) { - $options += $metadata; - $this->info = $this->connection->patchDataset($options + $this->identity); + $options = $this->applyEtagHeader( + $options + + $metadata + + $this->identity + ); - return $this->info; + if (!isset($options['etag']) && !isset($options['retries'])) { + $options['retries'] = 0; + } + + return $this->info = $this->connection->patchDataset($options); } /** @@ -220,6 +245,9 @@ function (array $table) { /** * Creates a table. * + * Please note that by default the library will not attempt to retry this + * call on your behalf. + * * Example: * ``` * $table = $dataset->createTable('aTable'); @@ -243,11 +271,15 @@ public function createTable($id, array $options = []) unset($options['metadata']); } - $response = $this->connection->insertTable([ - 'projectId' => $this->identity['projectId'], - 'datasetId' => $this->identity['datasetId'], - 'tableReference' => $this->identity + ['tableId' => $id] - ] + $options); + $response = $this->connection->insertTable( + [ + 'projectId' => $this->identity['projectId'], + 'datasetId' => $this->identity['datasetId'], + 'tableReference' => $this->identity + ['tableId' => $id] + ] + + $options + + ['retries' => 0] + ); return new Table( $this->connection, diff --git a/src/BigQuery/Exception/JobException.php b/src/BigQuery/Exception/JobException.php new file mode 100644 index 000000000000..8cd22e977047 --- /dev/null +++ b/src/BigQuery/Exception/JobException.php @@ -0,0 +1,46 @@ +job = $job; + parent::__construct($message, 0); + } + + /** + * Returns the job instance associated with the failure. + * + * @return Job + */ + public function getJob() + { + return $this->job; + } +} diff --git a/src/BigQuery/ExtractJobConfiguration.php b/src/BigQuery/ExtractJobConfiguration.php new file mode 100644 index 000000000000..06801edb5b7b --- /dev/null +++ b/src/BigQuery/ExtractJobConfiguration.php @@ -0,0 +1,165 @@ +dataset('my_dataset') + * ->table('my_table'); + * $extractJobConfig = $table->extract('gs://my_bucket/target.csv'); + * ``` + */ +class ExtractJobConfiguration implements JobConfigurationInterface +{ + use JobConfigurationTrait; + + /** + * @param string $projectId The project's ID. + * @param array $config A set of configuration options for a job. + */ + public function __construct($projectId, array $config) + { + $this->jobConfigurationProperties($projectId, $config); + } + + /** + * Sets the compression type to use for exported files. + * + * Example: + * ``` + * $extractJobConfig->compression('GZIP'); + * ``` + * + * @param string $compression The compression type. Acceptable values + * include `"GZIP"`, `"NONE"`. **Defaults to** `"NONE"`. + * @return ExtractJobConfiguration + */ + public function compression($compression) + { + $this->config['configuration']['extract']['compression'] = $compression; + + return $this; + } + + /** + * Sets the exported file format. Tables with nested or repeated fields + * cannot be exported as CSV. + * + * Example: + * ``` + * $extractJobConfig->destinationFormat('NEWLINE_DELIMITED_JSON'); + * ``` + * + * @param string $destinationFormat The exported file format. Acceptable + * values include `"CSV"`, `"NEWLINE_DELIMITED_JSON"`, `"AVRO"`. + * **Defaults to** `"CSV"`. + * @return ExtractJobConfiguration + */ + public function destinationFormat($destinationFormat) + { + $this->config['configuration']['extract']['destinationFormat'] = $destinationFormat; + + return $this; + } + + /** + * Sets a list of fully-qualified Google Cloud Storage URIs where the + * extracted table should be written. + * + * Example: + * ``` + * $extractJobConfig->destinationUris([ + * 'gs://my_bucket/destination.csv' + * ]); + * ``` + * + * @param array $destinationUris The destination URIs. + * @return ExtractJobConfiguration + */ + public function destinationUris(array $destinationUris) + { + $this->config['configuration']['extract']['destinationUris'] = $destinationUris; + + return $this; + } + + /** + * Sets the delimiter to use between fields in the exported data. + * + * Example: + * ``` + * $extractJobConfig->fieldDelimiter(','); + * ``` + * + * @param string $fieldDelimiter The field delimiter. **Defaults to** `","`. + * @return ExtractJobConfiguration + */ + public function fieldDelimiter($fieldDelimiter) + { + $this->config['configuration']['extract']['fieldDelimiter'] = $fieldDelimiter; + + return $this; + } + + /** + * Sets whether or not to print out a header row in the results. + * + * Example: + * ``` + * $extractJobConfig->printHeader(false); + * ``` + * + * @param bool $printHeader Whether or not to print out a header row. + * **Defaults to** `true`. + * @return ExtractJobConfiguration + */ + public function printHeader($printHeader) + { + $this->config['configuration']['extract']['printHeader'] = $printHeader; + + return $this; + } + + /** + * Sets a reference to the table being exported. + * + * Example: + * ``` + * $table = $bigQuery->dataset('my_dataset') + * ->table('my_table'); + * $extractJobConfig->sourceTable($table); + * ``` + * + * @param Table $sourceTable + * @return ExtractJobConfiguration + */ + public function sourceTable(Table $sourceTable) + { + $this->config['configuration']['extract']['sourceTable'] = $sourceTable->identity(); + + return $this; + } +} diff --git a/src/BigQuery/Job.php b/src/BigQuery/Job.php index 3c04f3552710..41eedfc55358 100644 --- a/src/BigQuery/Job.php +++ b/src/BigQuery/Job.php @@ -18,6 +18,8 @@ namespace Google\Cloud\BigQuery; use Google\Cloud\BigQuery\Connection\ConnectionInterface; +use Google\Cloud\BigQuery\Exception\JobException; +use Google\Cloud\Core\ArrayTrait; use Google\Cloud\Core\Exception\NotFoundException; /** @@ -27,6 +29,11 @@ */ class Job { + use ArrayTrait; + use JobWaitTrait; + + const MAX_RETRIES = 100; + /** * @var ConnectionInterface Represents a connection to BigQuery. */ @@ -96,6 +103,9 @@ public function exists() * Requests that a job be cancelled. You will need to poll the job to ensure * the cancel request successfully goes through. * + * Please note that by default the library will not attempt to retry this + * call on your behalf. + * * Example: * ``` * $job->cancel(); @@ -117,7 +127,11 @@ public function exists() */ public function cancel(array $options = []) { - $this->connection->cancelJob($options + $this->identity); + $this->connection->cancelJob( + $options + + ['retries' => 0] + + $this->identity + ); } /** @@ -143,15 +157,45 @@ public function cancel(array $options = []) */ public function queryResults(array $options = []) { - $response = $this->connection->getQueryResults($options + $this->identity); - return new QueryResults( $this->connection, $this->identity['jobId'], $this->identity['projectId'], - $response, - $options, - $this->mapper + $this->connection->getQueryResults($options + $this->identity), + $this->mapper, + $this + ); + } + + /** + * Blocks until the job is complete. + * + * Example: + * ``` + * $job->waitUntilComplete(); + * ``` + * + * @param array $options [optional] { + * Configuration options. + * + * @type int $maxRetries The number of times to retry, checking if the + * job has completed. **Defaults to** `100`. + * } + * @throws JobException If the maximum number of retries while waiting for + * job completion has been exceeded. + */ + public function waitUntilComplete(array $options = []) + { + $maxRetries = $this->pluck('maxRetries', $options, false); + $this->wait( + function () use ($options) { + return $this->isComplete($options); + }, + function () use ($options) { + return $this->reload($options); + }, + $this, + $maxRetries ); } @@ -171,6 +215,7 @@ public function queryResults(array $options = []) * * echo 'Query complete!'; * ``` + * * @param array $options [optional] Configuration options. * @return bool */ diff --git a/src/BigQuery/JobConfigurationInterface.php b/src/BigQuery/JobConfigurationInterface.php new file mode 100644 index 000000000000..d6f470dc6301 --- /dev/null +++ b/src/BigQuery/JobConfigurationInterface.php @@ -0,0 +1,31 @@ +config = array_replace_recursive([ + 'projectId' => $projectId, + 'jobReference' => ['projectId' => $projectId] + ], $config); + + if (!isset($this->config['jobReference']['jobId'])) { + $this->config['jobReference']['jobId'] = $this->generateJobId(); + } + } + + /** + * Specifies the default dataset to use for unqualified table names in the + * query. + * + * @param bool $dryRun + * @return JobConfigInterface + */ + public function dryRun($dryRun) + { + $this->config['configuration']['dryRun'] = $dryRun; + + return $this; + } + + /** + * Sets a prefix for the job ID. + * + * @param string $jobIdPrefix If provided, the job ID will be of format + * `{$jobIdPrefix}-{jobId}`. + * @return JobConfigInterface + */ + public function jobIdPrefix($jobIdPrefix) + { + $this->jobIdPrefix = $jobIdPrefix; + + return $this; + } + + /** + * Specifies the default dataset to use for unqualified table names in the + * query. + * + * @param array $labels + * @return JobConfigInterface + */ + public function labels(array $labels) + { + $this->config['configuration']['labels'] = $labels; + + return $this; + } + /** - * Builds a configuration for a job. + * Returns the job config as an array. * - * @param string $name - * @param string $projectId - * @param array $config - * @param array $userDefinedOptions + * @access private * @return array */ - public function buildJobConfig($name, $projectId, array $config, array $userDefinedOptions) + public function toArray() { - if (isset($userDefinedOptions['jobConfig'])) { - $config = $userDefinedOptions['jobConfig'] + $config; + if ($this->jobIdPrefix) { + $this->config['jobReference']['jobId'] = sprintf( + '%s-%s', + $this->jobIdPrefix, + $this->config['jobReference']['jobId'] + ); } - unset($userDefinedOptions['jobConfig']); + return $this->config; + } - return [ - 'projectId' => $projectId, - 'configuration' => [ - $name => $config - ] - ] + $userDefinedOptions; + /** + * Generate a Job ID. + * + * @return string + */ + protected function generateJobId() + { + return (string) Uuid::uuid4(); } } diff --git a/src/BigQuery/JobWaitTrait.php b/src/BigQuery/JobWaitTrait.php new file mode 100644 index 000000000000..fb74af442eca --- /dev/null +++ b/src/BigQuery/JobWaitTrait.php @@ -0,0 +1,62 @@ +execute($retryFn); + } + } +} diff --git a/src/BigQuery/LoadJobConfiguration.php b/src/BigQuery/LoadJobConfiguration.php new file mode 100644 index 000000000000..7e38b2c8a5f0 --- /dev/null +++ b/src/BigQuery/LoadJobConfiguration.php @@ -0,0 +1,526 @@ +dataset('my_dataset') + * ->table('my_table'); + * $loadJobConfig = $table->load(fopen('/path/to/my/data.csv', 'r')); + * ``` + */ +class LoadJobConfiguration implements JobConfigurationInterface +{ + use JobConfigurationTrait; + + /** + * @param string $projectId The project's ID. + * @param array $config A set of configuration options for a job. + */ + public function __construct($projectId, array $config) + { + $this->jobConfigurationProperties($projectId, $config); + } + + /** + * Sets whether to accept rows that are missing trailing optional columns. + * The missing values are treated as nulls. If false, records with missing + * trailing columns are treated as bad records, and if there are too many + * bad records, an invalid error is returned in the job result. Only + * applicable to CSV, ignored for other formats. + * + * Example: + * ``` + * $loadJobConfig->allowJaggedRows(true); + * ``` + * + * @param bool $allowJaggedRows Whether or not to allow jagged rows. + * **Defaults to** `false`. + * @return LoadJobConfiguration + */ + public function allowJaggedRows($allowJaggedRows) + { + $this->config['configuration']['load']['allowJaggedRows'] = $allowJaggedRows; + + return $this; + } + + /** + * Sets whether quoted data sections that contain newline characters in a + * CSV file are allowed. + * + * Example: + * ``` + * $loadJobConfig->allowQuotedNewlines(true); + * ``` + * + * @param bool $allowQuotedNewlines Whether or not to allow quoted new + * lines. **Defaults to** `false`. + * @return LoadJobConfiguration + */ + public function allowQuotedNewlines($allowQuotedNewlines) + { + $this->config['configuration']['load']['allowQuotedNewlines'] = $allowQuotedNewlines; + + return $this; + } + + /** + * Sets whether we should automatically infer the options and schema for CSV + * and JSON sources. + * + * Example: + * ``` + * $loadJobConfig->autodetect(true); + * ``` + * + * @param bool $autodetect Whether or not to autodetect options and schema. + * @return LoadJobConfiguration + */ + public function autodetect($autodetect) + { + $this->config['configuration']['load']['autodetect'] = $autodetect; + + return $this; + } + + /** + * Set whether the job is allowed to create new tables. Creation, truncation + * and append actions occur as one atomic update upon job completion. + * + * Example: + * ``` + * $loadJobConfig->createDisposition('CREATE_NEVER'); + * ``` + * + * @param string $createDisposition The create disposition. Acceptable + * values include `"CREATED_IF_NEEDED"`, `"CREATE_NEVER"`. **Defaults + * to** `"CREATE_IF_NEEDED"`. + * @return LoadJobConfiguration + */ + public function createDisposition($createDisposition) + { + $this->config['configuration']['load']['createDisposition'] = $createDisposition; + + return $this; + } + + /** + * Sets the custom encryption configuration (e.g., Cloud KMS keys). + * + * Example: + * ``` + * $loadJobConfig->destinationEncryptionConfiguration([ + * 'kmsKeyName' => 'my_key' + * ]); + * ``` + * + * @param array $configuration Custom encryption configuration. + * @return LoadJobConfiguration + */ + public function destinationEncryptionConfiguration(array $configuration) + { + $this->config['configuration']['load']['destinationEncryptionConfiguration'] = $configuration; + + return $this; + } + + /** + * The data to be loaded into the table. + * + * Example: + * ``` + * $loadJobConfig->data(fopen('/path/to/my/data.csv', 'r')); + * ``` + * + * @param string|resource|StreamInterface $data The data to be loaded into + * the table. + * @return LoadJobConfiguration + */ + public function data($data) + { + $this->config['data'] = $data; + + return $this; + } + + /** + * Sets the destination table to load the data into. + * + * Example: + * ``` + * $table = $bigQuery->dataset('my_dataset') + * ->table('my_table'); + * $loadJobConfig->destinationTable($table); + * ``` + * + * @param Table $destinationTable The destination table. + * @return LoadJobConfiguration + */ + public function destinationTable(Table $destinationTable) + { + $this->config['configuration']['load']['destinationTable'] = $destinationTable->identity(); + + return $this; + } + + /** + * Sets the character encoding of the data. BigQuery decodes the data after + * the raw, binary data has been split using the values of the quote and + * fieldDelimiter properties. + * + * Example: + * ``` + * $loadJobConfig->encoding('UTF-8'); + * ``` + * + * @param string $encoding The encoding type. Acceptable values include + * `"UTF-8"`, `"ISO-8859-1"`. **Defaults to** `"UTF-8"`. + * @return LoadJobConfiguration + */ + public function encoding($encoding) + { + $this->config['configuration']['load']['encoding'] = $encoding; + + return $this; + } + + /** + * Sets the separator for fields in a CSV file. The separator can be any + * ISO-8859-1 single-byte character. To use a character in the range + * 128-255, you must encode the character as UTF8. BigQuery converts the + * string to ISO-8859-1 encoding, and then uses the first byte of the + * encoded string to split the data in its raw, binary state. BigQuery also + * supports the escape sequence "\t" to specify a tab separator. + * + * Example: + * ``` + * $loadJobConfig->fieldDelimiter('\t'); + * ``` + * + * @param string $fieldDelimiter The field delimiter. **Defaults to** `","`. + * @return LoadJobConfiguration + */ + public function fieldDelimiter($fieldDelimiter) + { + $this->config['configuration']['load']['fieldDelimiter'] = $fieldDelimiter; + + return $this; + } + + /** + * Sets whether values that are not represented in the table schema should + * be allowed. If true, the extra values are ignored. If false, records with + * extra columns are treated as bad records, and if there are too many bad + * records, an invalid error is returned in the job result. + * + * The sourceFormat property determines what BigQuery treats as an extra + * value: + * + * - CSV: Trailing columns. + * - JSON: Named values that don't match any column names. + * + * Example: + * ``` + * $loadJobConfig->ignoreUnknownValues(true); + * ``` + * + * @param bool $ignoreUnknownValues Whether or not to ignore unknown values. + * **Defaults to** `false`. + * @return LoadJobConfiguration + */ + public function ignoreUnknownValues($ignoreUnknownValues) + { + $this->config['configuration']['load']['ignoreUnknownValues'] = $ignoreUnknownValues; + + return $this; + } + + /** + * Sets the maximum number of bad records that BigQuery can ignore when + * running the job. If the number of bad records exceeds this value, an + * invalid error is returned in the job result. + * + * Example: + * ``` + * $loadJobConfig->maxBadRecords(10); + * ``` + * + * @param int $maxBadRecords The maximum number of bad records. + * **Defaults to** `0` (requires all records to be valid). + * @return LoadJobConfiguration + */ + public function maxBadRecords($maxBadRecords) + { + $this->config['configuration']['load']['maxBadRecords'] = $maxBadRecords; + + return $this; + } + + /** + * Sets a string that represents a null value in a CSV file. For example, + * if you specify "\N", BigQuery interprets "\N" as a null value when + * loading a CSV file. The default value is the empty string. If you set + * this property to a custom value, BigQuery throws an error if an empty + * string is present for all data types except for STRING and BYTE. For + * STRING and BYTE columns, BigQuery interprets the empty string as an + * empty value. + * + * Example: + * ``` + * $loadJobConfig->nullMarker('\N'); + * ``` + * + * @param string $nullMarker The null marker. + * @return LoadJobConfiguration + */ + public function nullMarker($nullMarker) + { + $this->config['configuration']['load']['nullMarker'] = $nullMarker; + + return $this; + } + + /** + * Sets a list of projection fields. If sourceFormat is set to + * "DATASTORE_BACKUP", indicates which entity properties to load into + * BigQuery from a Cloud Datastore backup. Property names are case sensitive + * and must be top-level properties. If no properties are specified, + * BigQuery loads all properties. If any named property isn't found in the + * Cloud Datastore backup, an invalid error is returned in the job result. + * + * Example: + * ``` + * $loadJobConfig->projectionFields([ + * 'field_name' + * ]); + * ``` + * + * @param array $projectionFields The projection fields. + * @return LoadJobConfiguration + */ + public function projectionFields(array $projectionFields) + { + $this->config['configuration']['load']['projectionFields'] = $projectionFields; + + return $this; + } + + /** + * Sets the value that is used to quote data sections in a CSV file. + * BigQuery converts the string to ISO-8859-1 encoding, and then uses the + * first byte of the encoded string to split the data in its raw, binary + * state. If your data does not contain quoted sections, set the property + * value to an empty string. If your data contains quoted newline + * characters, you must also set the allowQuotedNewlines property to true. + * + * Example: + * ``` + * $loadJobConfig->quote('"'); + * ``` + * + * @param string $quote The quote value. **Defaults to** `"""` + * (double quotes). + * @return LoadJobConfiguration + */ + public function quote($quote) + { + $this->config['configuration']['load']['quote'] = $quote; + + return $this; + } + + /** + * Sets the schema for the destination table. The schema can be omitted if + * the destination table already exists, or if you're loading data from + * Google Cloud Datastore. + * + * Example: + * ``` + * $loadJobConfig->schema([ + * 'fields' => [ + * [ + * 'name' => 'col1', + * 'type' => 'STRING', + * ], + * [ + * 'name' => 'col2', + * 'type' => 'BOOL', + * ] + * ] + * ]); + * ``` + * + * @param array $schema The table schema. + * @return LoadJobConfiguration + */ + public function schema(array $schema) + { + $this->config['configuration']['load']['schema'] = $schema; + + return $this; + } + + /** + * Sets options to allow the schema of the destination table to be updated + * as a side effect of the query job. Schema update options are supported + * in two cases: when writeDisposition is `"WRITE_APPEND"`; when + * writeDisposition is `"WRITE_TRUNCATE"` and the destination table is a + * partition of a table, specified by partition decorators. For normal + * tables, `"WRITE_TRUNCATE"` will always overwrite the schema. + * + * Example: + * ``` + * $loadJobConfig->schemaUpdateOptions([ + * 'ALLOW_FIELD_ADDITION' + * ]); + * ``` + * + * @param array $schemaUpdateOptions Schema update options. Acceptable + * values include `"ALLOW_FIELD_ADDITION"` (allow adding a nullable + * field to the schema), `"ALLOW_FIELD_RELAXATION"` (allow relaxing + * a required field in the original schema to nullable). + * @return LoadJobConfiguration + */ + public function schemaUpdateOptions(array $schemaUpdateOptions) + { + $this->config['configuration']['load']['schemaUpdateOptions'] = $schemaUpdateOptions; + + return $this; + } + + /** + * Sets the number of rows at the top of a CSV file that BigQuery will skip + * when loading the data. This property is useful if you have header rows in + * the file that should be skipped. + * + * Example: + * ``` + * $loadJobConfig->skipLeadingRows(10); + * ``` + * + * @param int $skipLeadingRows The number of rows to skip. + * **Defaults to** `0`. + * @return LoadJobConfiguration + */ + public function skipLeadingRows($skipLeadingRows) + { + $this->config['configuration']['load']['skipLeadingRows'] = $skipLeadingRows; + + return $this; + } + + /** + * Sets the format of the data files. + * + * Example: + * ``` + * $loadJobConfig->sourceFormat('NEWLINE_DELIMITED_JSON'); + * ``` + * + * @param string $sourceFormat The source format. Acceptable values include + * `"CSV"`, `"DATASTORE_BACKUP"`. `"NEWLINE_DELIMITED_JSON"`, + * `"AVRO"`. **Defaults to** `"CSV"`. + * @return LoadJobConfiguration + */ + public function sourceFormat($sourceFormat) + { + $this->config['configuration']['load']['sourceFormat'] = $sourceFormat; + + return $this; + } + + /** + * Sets the fully-qualified URIs that point to your data in Google Cloud. + * + * - For Google Cloud Storage URIs: Each URI can contain one '*' wildcard + * character and it must come after the 'bucket' name. + * - For Google Cloud Bigtable URIs: Exactly one URI can be specified and it + * has be a fully specified and valid HTTPS URL for a Google Cloud Bigtable + * table. + * - For Google Cloud Datastore backups: Exactly one URI can be specified. + * Also, the '*' wildcard character is not allowed. + * + * Example: + * ``` + * $loadJobConfig->sourceUris([ + * 'gs://my_bucket/source.csv' + * ]); + * ``` + * + * @param array $sourceUris The source URIs. + * @return LoadJobConfiguration + */ + public function sourceUris(array $sourceUris) + { + $this->config['configuration']['load']['sourceUris'] = $sourceUris; + + return $this; + } + + /** + * Sets time-based partitioning for the destination table. + * + * Example: + * ``` + * $loadJobConfig->timePartitioning([ + * 'type' => 'DAY' + * ]); + * ``` + * + * @param array $timePartitioning Time-based partitioning configuration. + * @return LoadJobConfiguration + */ + public function timePartitioning(array $timePartitioning) + { + $this->config['configuration']['load']['timePartitioning'] = $timePartitioning; + + return $this; + } + + /** + * Sets the action that occurs if the destination table already exists. Each + * action is atomic and only occurs if BigQuery is able to complete the job + * successfully. Creation, truncation and append actions occur as one atomic + * update upon job completion. + * + * Example: + * ``` + * $loadJobConfig->writeDisposition('WRITE_TRUNCATE'); + * ``` + * + * @param string $writeDisposition The write disposition. Acceptable values + * include `"WRITE_TRUNCATE"`, `"WRITE_APPEND"`, `"WRITE_EMPTY"`. + * **Defaults to** `"WRITE_APPEND"`. + * @return LoadJobConfiguration + */ + public function writeDisposition($writeDisposition) + { + $this->config['configuration']['load']['writeDisposition'] = $writeDisposition; + + return $this; + } +} diff --git a/src/BigQuery/QueryJobConfiguration.php b/src/BigQuery/QueryJobConfiguration.php new file mode 100644 index 000000000000..f41a8ee16f17 --- /dev/null +++ b/src/BigQuery/QueryJobConfiguration.php @@ -0,0 +1,453 @@ +query('SELECT commit FROM `bigquery-public-data.github_repos.commits` LIMIT 100'); + * ``` + */ +class QueryJobConfiguration implements JobConfigurationInterface +{ + use JobConfigurationTrait; + + /** + * @var ValueMapper Maps values between PHP and BigQuery. + */ + private $mapper; + + /** + * @param ValueMapper $mapper Maps values between PHP and BigQuery. + * @param string $projectId The project's ID. + * @param array $config A set of configuration options for a job. + */ + public function __construct(ValueMapper $mapper, $projectId, array $config) + { + $this->mapper = $mapper; + $this->jobConfigurationProperties($projectId, $config); + + if (!isset($this->config['configuration']['query']['useLegacySql'])) { + $this->config['configuration']['query']['useLegacySql'] = false; + } + } + + /** + * Sets whether or not the query can produce arbitrarily large result + * tables at a slight cost in performance. Only applies to queries + * performed with legacy SQL dialect and requires a + * {@see Google\Cloud\BigQuery\QueryJobConfiguration::destinationTable()} to + * be set. + * + * Example: + * ``` + * $query->allowLargeResults(true); + * ``` + * + * @param bool $allowLargeResults Whether or not to allow large result sets. + * @return QueryJobConfiguration + */ + public function allowLargeResults($allowLargeResults) + { + $this->config['configuration']['query']['allowLargeResults'] = $allowLargeResults; + + return $this; + } + + /** + * Sets whether the job is allowed to create new tables. + * + * Example: + * ``` + * $query->createDisposition('CREATE_NEVER'); + * ``` + * + * @param string $createDisposition The create disposition. Acceptable + * values include `"CREATE_IF_NEEDED"`, `"CREATE_NEVER"`. + * @return QueryJobConfiguration + */ + public function createDisposition($createDisposition) + { + $this->config['configuration']['query']['createDisposition'] = $createDisposition; + + return $this; + } + + /** + * Sets the default dataset to use for unqualified table names in the query. + * + * Example: + * ``` + * $dataset = $bigQuery->dataset('my_dataset'); + * $query->defaultDataset($dataset); + * ``` + * + * @param Dataset $defaultDataset The default dataset. + * @return QueryJobConfiguration + */ + public function defaultDataset(Dataset $defaultDataset) + { + $this->config['configuration']['query']['defaultDataset'] = $defaultDataset->identity(); + + return $this; + } + + /** + * Sets the custom encryption configuration (e.g., Cloud KMS keys). + * + * Example: + * ``` + * $query->destinationEncryptionConfiguration([ + * 'kmsKeyName' => 'my_key' + * ]); + * ``` + * + * @param array $configuration Custom encryption configuration. + * @return QueryJobConfiguration + */ + public function destinationEncryptionConfiguration(array $configuration) + { + $this->config['configuration']['query']['destinationEncryptionConfiguration'] = $configuration; + + return $this; + } + + /** + * Sets the table where the query results should be stored. If not set, a + * new table will be created to store the results. This property must be set + * for large results that exceed the maximum response size. + * + * Example: + * ``` + * $table = $bigQuery->dataset('my_dataset') + * ->table('my_table'); + * $query->destinationTable($table); + * ``` + * + * @param Table $destinationTable The destination table. + * @return QueryJobConfiguration + */ + public function destinationTable(Table $destinationTable) + { + $this->config['configuration']['query']['destinationTable'] = $destinationTable->identity(); + + return $this; + } + + /** + * Sets whether or not to flatten all nested and repeated fields in the + * query results. Only applies to queries performed with legacy SQL dialect. + * {@see Google\Cloud\BigQuery\QueryJobConfiguration::allowLargeResults()} + * must be true if this is set to false. + * + * Example: + * ``` + * $query->useLegacySql(true) + * ->flattenResults(true); + * ``` + * + * @param bool $flattenResults Whether or not to flatten results. + * @return QueryJobConfiguration + */ + public function flattenResults($flattenResults) + { + $this->config['configuration']['query']['flattenResults'] = $flattenResults; + + return $this; + } + + /** + * Sets the billing tier limit for this job. Queries that have resource + * usage beyond this tier will fail (without incurring a charge). If + * unspecified, this will be set to your project default. + * + * Example: + * ``` + * $query->maximumBillingTier(1); + * ``` + * + * @see https://cloud.google.com/bigquery/pricing#high-compute High-Compute queries + * + * @param int $maximumBillingTier The maximum billing tier. + * @return QueryJobConfiguration + */ + public function maximumBillingTier($maximumBillingTier) + { + $this->config['configuration']['query']['maximumBillingTier'] = $maximumBillingTier; + + return $this; + } + + /** + * Sets a bytes billed limit for this job. Queries that will have bytes + * billed beyond this limit will fail (without incurring a charge). If + * unspecified, this will be set to your project default. + * + * Example: + * ``` + * $query->maximumBytesBilled(3000); + * ``` + * + * @param int $maximumBytesBilled The maximum allowable bytes to bill. + * @return QueryJobConfiguration + */ + public function maximumBytesBilled($maximumBytesBilled) + { + $this->config['configuration']['query']['maximumBytesBilled'] = $maximumBytesBilled; + + return $this; + } + + /** + * Sets parameters to be used on the query. Only available for standard SQL + * queries. + * + * For examples of usage please see + * {@see Google\Cloud\BigQuery\BigQueryClient::runQuery()}. + * + * @param array $parameters Parameters to use on the query. When providing + * a non-associative array positional parameters (`?`) will be used. + * When providing an associative array named parameters will be used + * (`@name`). + * @return QueryJobConfiguration + */ + public function parameters(array $parameters) + { + $queryParams = []; + $this->config['configuration']['query']['parameterMode'] = $this->isAssoc($parameters) + ? 'named' + : 'positional'; + + foreach ($parameters as $name => $value) { + $param = $this->mapper->toParameter($value); + + if ($this->config['configuration']['query']['parameterMode'] === 'named') { + $param += ['name' => $name]; + } + + $queryParams[] = $param; + } + + $this->config['configuration']['query']['queryParameters'] = $queryParams; + + return $this; + } + + /** + * Sets a priority for the query. + * + * Example: + * ``` + * $query->priority('BATCH'); + * ``` + * + * @param string $priority The priority. Acceptable values include + * `"INTERACTIVE"`, `"BATCH"`. **Defaults to** `"INTERACTIVE"`. + * @return QueryJobConfiguration + */ + public function priority($priority) + { + $this->config['configuration']['query']['priority'] = $priority; + + return $this; + } + + /** + * Sets the SQL query. + * + * Example: + * ``` + * $query->query( + * 'SELECT commit FROM `bigquery-public-data.github_repos.commits` LIMIT 100' + * ); + * ``` + * + * @param string $query SQL query text to execute. + * @return QueryJobConfiguration + */ + public function query($query) + { + $this->config['configuration']['query']['query'] = $query; + + return $this; + } + + /** + * Sets options to allow the schema of the destination table to be updated + * as a side effect of the query job. Schema update options are supported + * in two cases: when writeDisposition is `"WRITE_APPEND"`; when + * writeDisposition is `"WRITE_TRUNCATE"` and the destination table is a + * partition of a table, specified by partition decorators. For normal + * tables, `"WRITE_TRUNCATE"` will always overwrite the schema. + * + * Example: + * ``` + * $query->schemaUpdateOptions([ + * 'ALLOW_FIELD_ADDITION' + * ]); + * ``` + * + * @param array $schemaUpdateOptions Schema update options. Acceptable + * values include `"ALLOW_FIELD_ADDITION"` (allow adding a nullable + * field to the schema), `"ALLOW_FIELD_RELAXATION"` (allow relaxing + * a required field in the original schema to nullable). + * @return QueryJobConfiguration + */ + public function schemaUpdateOptions(array $schemaUpdateOptions) + { + $this->config['configuration']['query']['schemaUpdateOptions'] = $schemaUpdateOptions; + + return $this; + } + + /** + * Sets table definitions for querying an external data source outside of + * BigQuery. Describes the data format, location and other properties of the + * data source. + * + * Example: + * ``` + * $query->tableDefinitions([ + * 'autodetect' => true, + * 'sourceUris' => [ + * 'gs://my_bucket/table.json' + * ] + * ]); + * ``` + * + * @param array $tableDefinitions The table definitions. + * @return QueryJobConfiguration + */ + public function tableDefinitions(array $tableDefinitions) + { + $this->config['configuration']['query']['tableDefinitions'] = $tableDefinitions; + + return $this; + } + + /** + * Sets time-based partitioning for the destination table. + * + * Example: + * ``` + * $query->timePartitioning([ + * 'type' => 'DAY' + * ]); + * ``` + * + * @param array $timePartitioning Time-based partitioning configuration. + * @return QueryJobConfiguration + */ + public function timePartitioning(array $timePartitioning) + { + $this->config['configuration']['query']['timePartitioning'] = $timePartitioning; + + return $this; + } + + /** + * Sets whether or not to use legacy SQL dialect. When not set, defaults to + * false in this client. + * + * Example: + * ``` + * $query->useLegacySql(true); + * ``` + * + * @param bool $useLegacySql Whether or not to use legacy SQL dialect. + * @return QueryJobConfiguration + */ + public function useLegacySql($useLegacySql) + { + $this->config['configuration']['query']['useLegacySql'] = $useLegacySql; + + return $this; + } + + /** + * Sets whether or not to use the query cache. + * + * Example: + * ``` + * $query->useQueryCache(true); + * ``` + * + * @see https://cloud.google.com/bigquery/querying-data#query-caching Query Caching + * + * @param bool $useQueryCache Whether or not to use the query cache. + * @return QueryJobConfiguration + */ + public function useQueryCache($useQueryCache) + { + $this->config['configuration']['query']['useQueryCache'] = $useQueryCache; + + return $this; + } + + /** + * Sets user-defined function resources used in the query. + * + * Example: + * ``` + * $query->userDefinedFunctionResources([ + * ['resourceUri' => 'gs://my_bucket/code_path'] + * ]); + * ``` + * + * @param array $userDefinedFunctionResources User-defined function + * resources used in the query. This is to be formatted as a list of + * sub-arrays containing either a key `resourceUri` or `inlineCode`. + * @return QueryJobConfiguration + */ + public function userDefinedFunctionResources(array $userDefinedFunctionResources) + { + $this->config['configuration']['query']['userDefinedFunctionResources'] = $userDefinedFunctionResources; + + return $this; + } + + /** + * Sets the action that occurs if the destination table already exists. Each + * action is atomic and only occurs if BigQuery is able to complete the job + * successfully. Creation, truncation and append actions occur as one atomic + * update upon job completion. + * + * Example: + * ``` + * $query->writeDisposition('WRITE_TRUNCATE'); + * ``` + * + * @param string $writeDisposition The write disposition. Acceptable values + * include `"WRITE_TRUNCATE"`, `"WRITE_APPEND"`, `"WRITE_EMPTY"`. + * **Defaults to** `"WRITE_EMPTY"`. + * @return QueryJobConfiguration + */ + public function writeDisposition($writeDisposition) + { + $this->config['configuration']['query']['writeDisposition'] = $writeDisposition; + + return $this; + } +} diff --git a/src/BigQuery/QueryResults.php b/src/BigQuery/QueryResults.php index d1b7e5f6e866..20e66b5984b9 100644 --- a/src/BigQuery/QueryResults.php +++ b/src/BigQuery/QueryResults.php @@ -18,6 +18,7 @@ namespace Google\Cloud\BigQuery; use Google\Cloud\BigQuery\Connection\ConnectionInterface; +use Google\Cloud\Core\ArrayTrait; use Google\Cloud\Core\Exception\GoogleException; use Google\Cloud\Core\Iterator\ItemIterator; use Google\Cloud\Core\Iterator\PageIterator; @@ -30,8 +31,11 @@ * calling {@see Google\Cloud\BigQuery\BigQueryClient::runQuery()} or * {@see Google\Cloud\BigQuery\Job::queryResults()}. */ -class QueryResults +class QueryResults implements \IteratorAggregate { + use ArrayTrait; + use JobWaitTrait; + /** * @var ConnectionInterface Represents a connection to BigQuery. */ @@ -48,14 +52,14 @@ class QueryResults private $info; /** - * @var ValueMapper $mapper Maps values between PHP and BigQuery. + * @var Job The job from which the query results originated. */ - private $mapper; + private $job; /** - * @var array The options to use when reloading query data. + * @var ValueMapper Maps values between PHP and BigQuery. */ - private $reloadOptions; + private $mapper; /** * @param ConnectionInterface $connection Represents a connection to @@ -63,31 +67,30 @@ class QueryResults * @param string $jobId The job's ID. * @param string $projectId The project's ID. * @param array $info The query result's metadata. - * @param array $reloadOptions The options to use when reloading query data. * @param ValueMapper $mapper Maps values between PHP and BigQuery. + * @param Job $job The job from which the query results originated. */ public function __construct( ConnectionInterface $connection, $jobId, $projectId, array $info, - array $reloadOptions, - ValueMapper $mapper + ValueMapper $mapper, + Job $job ) { $this->connection = $connection; $this->info = $info; - $this->reloadOptions = $reloadOptions; $this->identity = [ 'jobId' => $jobId, 'projectId' => $projectId ]; $this->mapper = $mapper; + $this->job = $job; } /** * Retrieves the rows associated with the query and merges them together - * with the table's schema. It is recommended to check the completeness of - * the query before attempting to access rows. + * with the table's schema. * * Refer to the table below for a guide on how BigQuery types are mapped as * they come back from the API. @@ -109,27 +112,31 @@ public function __construct( * * Example: * ``` - * $isComplete = $queryResults->isComplete(); - * - * if ($isComplete) { - * $rows = $queryResults->rows(); + * $rows = $queryResults->rows(); * - * foreach ($rows as $row) { - * echo $row['name'] . PHP_EOL; - * } + * foreach ($rows as $row) { + * echo $row['name'] . PHP_EOL; * } * ``` * - * @param array $options [optional] Configuration options. + * @param array $options [optional] { + * Configuration options. + * + * @type int $maxResults Maximum number of results to read per page. + * @type int $startIndex Zero-based index of the starting row. + * @type int $timeoutMs If not yet complete, how long to wait in + * milliseconds. **Defaults to** `10000` milliseconds (10 seconds). + * @type int $maxRetries The number of times to retry, checking if the + * query has completed. **Defaults to** `100`. + * } * @return ItemIterator - * @throws GoogleException Thrown if the query has not yet completed. + * @throws JobException If the maximum number of retries while waiting for + * query completion has been exceeded. + * @throws GoogleException Thrown in the case of a malformed response. */ public function rows(array $options = []) { - if (!$this->isComplete()) { - throw new GoogleException('The query has not completed yet.'); - } - + $this->waitUntilComplete($options); $schema = $this->info['schema']['fields']; return new ItemIterator( @@ -164,27 +171,40 @@ function (array $row) use ($schema) { } /** - * Checks the query's completeness. Useful in combination with - * {@see Google\Cloud\BigQuery\QueryResults::reload()} to poll for query status. + * Blocks until the query is complete. * * Example: * ``` - * $isComplete = $queryResults->isComplete(); - * - * while (!$isComplete) { - * sleep(1); // small delay between requests - * $queryResults->reload(); - * $isComplete = $queryResults->isComplete(); - * } - * - * echo 'Query complete!'; + * $queryResults->waitUntilComplete(); * ``` * - * @return bool + * @param array $options [optional] { + * Configuration options. + * + * @type int $maxResults Maximum number of results to read per page. + * @type int $startIndex Zero-based index of the starting row. + * @type int $timeoutMs If not yet complete, how long to wait for the + * query to complete, in milliseconds. **Defaults to** + * `10000` milliseconds (10 seconds). + * @type int $maxRetries The number of times to retry, checking if the + * query has completed. **Defaults to** `100`. + * } + * @throws JobException If the maximum number of retries while waiting for + * query completion has been exceeded. */ - public function isComplete() + public function waitUntilComplete(array $options = []) { - return $this->info['jobComplete']; + $maxRetries = $this->pluck('maxRetries', $options, false); + $this->wait( + function () { + return $this->isComplete(); + }, + function () use ($options) { + return $this->reload($options); + }, + $this->job, + $maxRetries + ); } /** @@ -209,20 +229,9 @@ public function info() /** * Triggers a network request to reload the query's details. * - * Useful when needing to poll an incomplete query - * for status. Configuration options will be inherited from - * {@see Google\Cloud\BigQuery\Job::queryResults()} or - * {@see Google\Cloud\BigQuery\BigQueryClient::runQuery()}, but they can be - * overridden if needed. - * * Example: * ``` - * $queryResults->isComplete(); // returns false - * sleep(1); // let's wait for a moment... - * $queryResults->reload(); // executes a network request - * if ($queryResults->isComplete()) { - * echo "Query complete!"; - * } + * $queryResults->reload(); * ``` * * @see https://cloud.google.com/bigquery/docs/reference/v2/jobs/getQueryResults @@ -240,8 +249,9 @@ public function info() */ public function reload(array $options = []) { - $options += $this->identity; - return $this->info = $this->connection->getQueryResults($options + $this->reloadOptions); + return $this->info = $this->connection->getQueryResults( + $options + $this->identity + ); } /** @@ -260,4 +270,59 @@ public function identity() { return $this->identity; } + + /** + * Checks the job's completeness. Useful in combination with + * {@see Google\Cloud\BigQuery\QueryResults::reload()} to poll for query + * status. + * + * Example: + * ``` + * $isComplete = $queryResults->isComplete(); + * + * while (!$isComplete) { + * sleep(1); // let's wait for a moment... + * $queryResults->reload(); + * $isComplete = $queryResults->isComplete(); + * } + * + * echo 'Query complete!'; + * ``` + * + * @return bool + */ + public function isComplete() + { + return $this->info['jobComplete']; + } + + /** + * Returns a reference to the {@see Google\Cloud\BigQuery\Job} instance used + * to fetch the query results. This is especially useful when attempting to + * access job statistics after calling + * {@see Google\Cloud\BigQuery\BigQueryClient::runQuery()}. + * + * Example: + * ``` + * $job = $queryResults->job(); + * ``` + * + * @return array + */ + public function job() + { + return $this->job; + } + + /** + * @access private + * @return ItemIterator + * @throws JobException If the maximum number of retries while waiting for + * query completion has been exceeded. + * @throws GoogleException Thrown in the case of a malformed response. + */ + public function getIterator() + { + return $this->rows(); + } } diff --git a/src/BigQuery/Table.php b/src/BigQuery/Table.php index 5e2dc74aa1e9..8a419cb27f71 100644 --- a/src/BigQuery/Table.php +++ b/src/BigQuery/Table.php @@ -19,9 +19,12 @@ use Google\Cloud\BigQuery\Connection\ConnectionInterface; use Google\Cloud\Core\ArrayTrait; +use Google\Cloud\Core\ConcurrencyControlTrait; +use Google\Cloud\Core\Exception\ConflictException; use Google\Cloud\Core\Exception\NotFoundException; use Google\Cloud\Core\Iterator\ItemIterator; use Google\Cloud\Core\Iterator\PageIterator; +use Google\Cloud\Core\RetryDeciderTrait; use Google\Cloud\Storage\StorageObject; use Psr\Http\Message\StreamInterface; @@ -32,8 +35,12 @@ */ class Table { + const MAX_RETRIES = 100; + const INSERT_CREATE_MAX_DELAY_MICROSECONDS = 60000000; + use ArrayTrait; - use JobConfigurationTrait; + use ConcurrencyControlTrait; + use RetryDeciderTrait; /** * @var ConnectionInterface Represents a connection to BigQuery. @@ -80,6 +87,11 @@ public function __construct( 'datasetId' => $datasetId, 'projectId' => $projectId ]; + $this->setHttpRetryCodes([]); + $this->setHttpRetryMessages([ + 'rateLimitExceeded', + 'backendError' + ]); } /** @@ -108,6 +120,9 @@ public function exists() /** * Delete the table. * + * Please note that by default the library will not attempt to retry this + * call on your behalf. + * * Example: * ``` * $table->delete(); @@ -119,12 +134,24 @@ public function exists() */ public function delete(array $options = []) { - $this->connection->deleteTable($options + $this->identity); + $this->connection->deleteTable( + $options + + ['retries' => 0] + + $this->identity + ); } /** * Update the table. * + * Providing an `etag` key as part of `$metadata` will enable simultaneous + * update protection. This is useful in preventing override of modifications + * made by another user. The resource's current etag can be obtained via a + * GET request on the resource. + * + * Please note that by default the library will not automatically retry this + * call on your behalf unless an `etag` is set. + * * Example: * ``` * $table->update([ @@ -133,6 +160,7 @@ public function delete(array $options = []) * ``` * * @see https://cloud.google.com/bigquery/docs/reference/v2/tables/patch Tables patch API documentation. + * @see https://cloud.google.com/bigquery/docs/api-performance#patch Patch (Partial Update) * * @param array $metadata The available options for metadata are outlined * at the [Table Resource API docs](https://cloud.google.com/bigquery/docs/reference/v2/tables#resource) @@ -140,10 +168,17 @@ public function delete(array $options = []) */ public function update(array $metadata, array $options = []) { - $options += $metadata; - $this->info = $this->connection->patchTable($options + $this->identity); + $options = $this->applyEtagHeader( + $options + + $metadata + + $this->identity + ); - return $this->info; + if (!isset($options['etag']) && !isset($options['retries'])) { + $options['retries'] = 0; + } + + return $this->info = $this->connection->patchTable($options); } /** @@ -211,45 +246,64 @@ function (array $row) use ($schema) { } /** - * Runs a copy job which copies this table to a specified destination table. + * Starts a job in an synchronous fashion, waiting for the job to complete + * before returning. * * Example: * ``` - * $sourceTable = $bigQuery->dataset('myDataset')->table('mySourceTable'); - * $destinationTable = $bigQuery->dataset('myDataset')->table('myDestinationTable'); - * - * $job = $sourceTable->copy($destinationTable); + * $job = $table->runJob($jobConfig); + * echo $job->isComplete(); // true * ``` * * @see https://cloud.google.com/bigquery/docs/reference/v2/jobs Jobs insert API Documentation. * - * @param Table $destination The destination table. + * @param JobConfigurationInterface $config The job configuration. * @param array $options [optional] { * Configuration options. * - * @type array $jobConfig Configuration settings for a copy job are - * outlined in the [API Docs for `configuration.copy`](https://goo.gl/m8dro9). - * If not provided default settings will be used. + * @type int $maxRetries The number of times to retry, checking if the + * job has completed. **Defaults to** `100`. * } * @return Job */ - public function copy(Table $destination, array $options = []) + public function runJob(JobConfigurationInterface $config, array $options = []) { - $config = $this->buildJobConfig( - 'copy', - $this->identity['projectId'], - [ - 'destinationTable' => $destination->identity(), - 'sourceTable' => $this->identity - ], - $options - ); + $maxRetries = $this->pluck('maxRetries', $options, false); + $job = $this->startJob($config, $options); + $job->waitUntilComplete(['maxRetries' => $maxRetries]); + + return $job; + } + + /** + * Starts a job in an asynchronous fashion. In this case, it will be + * required to manually trigger a call to wait for job completion. + * + * Example: + * ``` + * $job = $table->startJob($jobConfig); + * ``` + * + * @see https://cloud.google.com/bigquery/docs/reference/v2/jobs Jobs insert API Documentation. + * + * @param JobConfigurationInterface $config The job configuration. + * @param array $options [optional] Configuration options. + * @return Job + */ + public function startJob(JobConfigurationInterface $config, array $options = []) + { + $response = null; + $config = $config->toArray() + $options; - $response = $this->connection->insertJob($config); + if (isset($config['data'])) { + $response = $this->connection->insertJobUpload($config)->upload(); + } else { + $response = $this->connection->insertJob($config); + } return new Job( $this->connection, - $response['jobReference']['jobId'], + $config['jobReference']['jobId'], $this->identity['projectId'], $this->mapper, $response @@ -257,13 +311,52 @@ public function copy(Table $destination, array $options = []) } /** - * Runs an extract job which exports the contents of a table to Cloud - * Storage. + * Returns a copy job configuration to be passed to either + * {@see Google\Cloud\BigQuery\Table::runJob()} or + * {@see Google\Cloud\BigQuery\Table::startJob()}. A + * configuration can be built using fluent setters or by providing a full + * set of options at once. + * + * Example: + * ``` + * $sourceTable = $bigQuery->dataset('myDataset') + * ->table('mySourceTable'); + * $destinationTable = $bigQuery->dataset('myDataset') + * ->table('myDestinationTable'); + * + * $copyJobConfig = $sourceTable->copy($destinationTable); + * ``` + * + * @see https://cloud.google.com/bigquery/docs/reference/v2/jobs Jobs insert API Documentation. + * + * @param Table $destination The destination table. + * @param array $options [optional] Please see the + * [upstream API documentation for Job configuration] + * (https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration) + * for the available options. + * @return CopyJobConfiguration + */ + public function copy(Table $destination, array $options = []) + { + return (new CopyJobConfiguration( + $this->identity['projectId'], + $options + )) + ->destinationTable($destination) + ->sourceTable($this); + } + + /** + * Returns an extract job configuration to be passed to either + * {@see Google\Cloud\BigQuery\Table::runJob()} or + * {@see Google\Cloud\BigQuery\Table::startJob()}. A + * configuration can be built using fluent setters or by providing a full + * set of options at once. * * Example: * ``` * $destinationObject = $storage->bucket('myBucket')->object('tableOutput'); - * $job = $table->export($destinationObject); + * $extractJobConfig = $table->extract($destinationObject); * ``` * * @see https://cloud.google.com/bigquery/docs/reference/v2/jobs Jobs insert API Documentation. @@ -272,96 +365,73 @@ public function copy(Table $destination, array $options = []) * a {@see Google\Cloud\Storage\StorageObject} or a URI pointing to * a Google Cloud Storage object in the format of * `gs://{bucket-name}/{object-name}`. - * @param array $options [optional] { - * Configuration options. - * - * @type array $jobConfig Configuration settings for an extract job are - * outlined in the [API Docs for `configuration.extract`](https://goo.gl/SQ9XAE). - * If not provided default settings will be used. - * } - * @return Job + * @param array $options [optional] Please see the + * [upstream API documentation for Job configuration] + * (https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration) + * for the available options. + * @return ExportJobConfiguration */ - public function export($destination, array $options = []) + public function extract($destination, array $options = []) { if ($destination instanceof StorageObject) { $destination = $destination->gcsUri(); } - $config = $this->buildJobConfig( - 'extract', + return (new ExtractJobConfiguration( $this->identity['projectId'], - [ - 'sourceTable' => $this->identity, - 'destinationUris' => [$destination] - ], $options - ); - - $response = $this->connection->insertJob($config); - - return new Job( - $this->connection, - $response['jobReference']['jobId'], - $this->identity['projectId'], - $this->mapper, - $response - ); + )) + ->destinationUris([$destination]) + ->sourceTable($this); } /** - * Runs a load job which loads the provided data into the table. + * Returns a load job configuration to be passed to either + * {@see Google\Cloud\BigQuery\Table::runJob()} or + * {@see Google\Cloud\BigQuery\Table::startJob()}. A + * configuration can be built using fluent setters or by providing a full + * set of options at once. * * Example: * ``` - * $job = $table->load(fopen('/path/to/my/data.csv', 'r')); + * $loadJobConfig = $table->load(fopen('/path/to/my/data.csv', 'r')); * ``` * * @see https://cloud.google.com/bigquery/docs/reference/v2/jobs Jobs insert API Documentation. * * @param string|resource|StreamInterface $data The data to load. - * @param array $options [optional] { - * Configuration options. - * - * @type array $jobConfig Configuration settings for a load job are - * outlined in the [API Docs for `configuration.load`](https://goo.gl/j6jyHv). - * If not provided default settings will be used. - * } - * @return Job + * @param array $options [optional] Please see the + * [upstream API documentation for Job configuration] + * (https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration) + * for the available options. + * @return LoadJobConfiguration */ public function load($data, array $options = []) { - $response = null; - $config = $this->buildJobConfig( - 'load', + $config = (new LoadJobConfiguration( $this->identity['projectId'], - ['destinationTable' => $this->identity], $options - ); + )) + ->destinationTable($this); if ($data) { - $config['data'] = $data; - $response = $this->connection->insertJobUpload($config)->upload(); - } else { - $response = $this->connection->insertJob($config); + $config->data($data); } - return new Job( - $this->connection, - $response['jobReference']['jobId'], - $this->identity['projectId'], - $this->mapper, - $response - ); + return $config; } /** - * Runs a load job which loads data from a file in a Storage bucket into the - * table. + * Returns a load job configuration to be passed to either + * {@see Google\Cloud\BigQuery\Table::runJob()} or + * {@see Google\Cloud\BigQuery\Table::startJob()}. A + * configuration can be built using fluent setters or by providing a full + * set of options at once. * * Example: * ``` * $object = $storage->bucket('myBucket')->object('important-data.csv'); - * $job = $table->load($object); + * $loadJobConfig = $table->loadFromStorage($object); * ``` * * @see https://cloud.google.com/bigquery/docs/reference/v2/jobs Jobs insert API Documentation. @@ -370,14 +440,11 @@ public function load($data, array $options = []) * a {@see Google\Cloud\Storage\StorageObject} or a URI pointing to a * Google Cloud Storage object in the format of * `gs://{bucket-name}/{object-name}`. - * @param array $options [optional] { - * Configuration options. - * - * @type array $jobConfig Configuration settings for a load job are - * outlined in the [API Docs for `configuration.load`](https://goo.gl/j6jyHv). - * If not provided default settings will be used. - * } - * @return Job + * @param array $options [optional] Please see the + * [upstream API documentation for Job configuration] + * (https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration) + * for the available options. + * @return LoadJobConfiguration */ public function loadFromStorage($object, array $options = []) { @@ -385,7 +452,7 @@ public function loadFromStorage($object, array $options = []) $object = $object->gcsUri(); } - $options['jobConfig']['sourceUris'] = [$object]; + $options['configuration']['load']['sourceUris'] = [$object]; return $this->load(null, $options); } @@ -393,6 +460,9 @@ public function loadFromStorage($object, array $options = []) /** * Insert a record into the table without running a load job. * + * Please note that by default the library will not automatically retry this + * call on your behalf unless an `insertId` is set. + * * Example: * ``` * $row = [ @@ -447,6 +517,9 @@ public function insertRow(array $row, array $options = []) /** * Insert records into the table without running a load job. * + * Please note that by default the library will not automatically retry this + * call on your behalf unless an `insertId` is set. + * * Example: * ``` * $rows = [ @@ -491,6 +564,19 @@ public function insertRow(array $row, array $options = []) * @param array $options [optional] { * Configuration options. * + * @type bool $autoCreate Whether or not to attempt to automatically + * create the table in the case it does not exist. Please note, it + * will be required to provide a schema through + * $tableMetadata['schema'] in the case the table does not already + * exist. **Defaults to** `false`. + * @type $tableMetadata Metadata to apply to table to be created. The + * full set of metadata are outlined at the + * [Table Resource API docs](https://cloud.google.com/bigquery/docs/reference/v2/tables#resource). + * Only applies when `autoCreate` is `true`. + * @type $maxRetries The maximum number of times to attempt creating the + * table in the case of failure. Please note, each retry attempt + * may take up to two minutes. Only applies when `autoCreate` is + * `true`. **Defaults to** `100`. * @type bool $skipInvalidRows Insert all valid rows of a request, even * if invalid rows exist. The default value is `false`, which * causes the entire request to fail if any invalid rows exist. @@ -508,16 +594,26 @@ public function insertRow(array $row, array $options = []) * for considerations when working with templates tables. * } * @return InsertResponse - * @throws \InvalidArgumentException + * @throws \InvalidArgumentException If a provided row does not contain a + * `data` key, if a schema is not defined when `autoCreate` is + * `true`, or if less than 1 row is provided. * @codingStandardsIgnoreEnd */ public function insertRows(array $rows, array $options = []) { + if (count($rows) === 0) { + throw new \InvalidArgumentException('Must provide at least a single row.'); + } + foreach ($rows as $row) { if (!isset($row['data'])) { throw new \InvalidArgumentException('A row must have a data key.'); } + if (!isset($options['retries']) && !isset($row['insertId'])) { + $options['retries'] = 0; + } + foreach ($row['data'] as $key => $item) { $row['data'][$key] = $this->mapper->toBigQuery($item); } @@ -528,7 +624,7 @@ public function insertRows(array $rows, array $options = []) } return new InsertResponse( - $this->connection->insertAllTableData($this->identity + $options), + $this->handleInsert($options), $options['rows'] ); } @@ -608,4 +704,68 @@ public function identity() { return $this->identity; } + + /** + * Delay execution in microseconds. + * + * @param int $microSeconds + */ + protected function usleep($microSeconds) + { + usleep($microSeconds); + } + + /** + * Handles inserting table data and manages custom retry logic in the case + * a table needs to be created. + * + * @param array $options Configuration options. + * @return array + */ + private function handleInsert(array $options) + { + $attempt = 0; + $metadata = $this->pluck('tableMetadata', $options, false) ?: []; + $autoCreate = $this->pluck('autoCreate', $options, false) ?: false; + $maxRetries = $this->pluck('maxRetries', $options, false) ?: self::MAX_RETRIES; + + while (true) { + try { + return $this->connection->insertAllTableData( + $this->identity + $options + ); + } catch (NotFoundException $ex) { + if ($autoCreate === true && $attempt <= $maxRetries) { + if (!isset($metadata['schema'])) { + throw new \InvalidArgumentException( + 'A schema is required when creating a table.' + ); + } + + $this->usleep(mt_rand(1, self::INSERT_CREATE_MAX_DELAY_MICROSECONDS)); + + try { + $this->connection->insertTable($metadata + [ + 'projectId' => $this->identity['projectId'], + 'datasetId' => $this->identity['datasetId'], + 'tableReference' => $this->identity, + 'retries' => 0 + ]); + } catch (ConflictException $ex) { + } catch (\Exception $ex) { + $retryFunction = $this->getRetryFunction(); + + if (!$retryFunction($ex)) { + throw $ex; + } + } + + $this->usleep(self::INSERT_CREATE_MAX_DELAY_MICROSECONDS); + $attempt++; + } else { + throw $ex; + } + } + } + } } diff --git a/src/BigQuery/composer.json b/src/BigQuery/composer.json index 6eded48fdc12..cf54b78f6b6b 100644 --- a/src/BigQuery/composer.json +++ b/src/BigQuery/composer.json @@ -4,7 +4,8 @@ "license": "Apache-2.0", "minimum-stability": "stable", "require": { - "google/cloud-core": "^1.0" + "google/cloud-core": "^1.0", + "ramsey/uuid": "~3" }, "suggest": { "google/cloud-storage": "Makes it easier to load data from Cloud Storage into BigQuery" diff --git a/src/Core/ArrayTrait.php b/src/Core/ArrayTrait.php index efca6404bf27..93736fe5c3cb 100644 --- a/src/Core/ArrayTrait.php +++ b/src/Core/ArrayTrait.php @@ -53,7 +53,7 @@ private function pluck($key, array &$arr, $isRequired = true) * * @param string $keys * @param array $arr - * @return string + * @return array */ private function pluckArray(array $keys, &$arr) { diff --git a/src/Core/ConcurrencyControlTrait.php b/src/Core/ConcurrencyControlTrait.php new file mode 100644 index 000000000000..aca1f9e7ead4 --- /dev/null +++ b/src/Core/ConcurrencyControlTrait.php @@ -0,0 +1,48 @@ + [], 'shouldSignRequest' => true, 'componentVersion' => null, - 'restRetryFunction' => null + 'restRetryFunction' => null, + 'restDelayFunction' => null ]; $this->componentVersion = $config['componentVersion']; @@ -111,6 +120,7 @@ public function __construct(array $config = []) $this->restOptions = $config['restOptions']; $this->shouldSignRequest = $config['shouldSignRequest']; $this->retryFunction = $config['restRetryFunction'] ?: $this->getRetryFunction(); + $this->delayFunction = $config['restDelayFunction']; } /** @@ -126,19 +136,17 @@ public function __construct(array $config = []) * **Defaults to** `3`. * @type callable $restRetryFunction Sets the conditions for whether or * not a request should attempt to retry. + * @type callable $restDelayFunction Sets the conditions for determining + * how long to wait between attempts to retry. * @type array $restOptions HTTP client specific configuration options. * } * @return ResponseInterface */ public function send(RequestInterface $request, array $options = []) { - $retries = isset($options['retries']) ? $options['retries'] : $this->retries; $restOptions = isset($options['restOptions']) ? $options['restOptions'] : $this->restOptions; $timeout = isset($options['requestTimeout']) ? $options['requestTimeout'] : $this->requestTimeout; - $retryFunction = isset($options['restRetryFunction']) - ? $options['restRetryFunction'] - : $this->retryFunction; - $backoff = new ExponentialBackoff($retries, $retryFunction); + $backoff = $this->configureBackoff($options); if ($timeout && !array_key_exists('timeout', $restOptions)) { $restOptions['timeout'] = $timeout; @@ -266,4 +274,30 @@ private function getExceptionMessage(\Exception $ex) return $ex->getMessage(); } + + /** + * Configures an exponential backoff implementation. + * + * @param array $options + * @return ExponentialBackoff + */ + private function configureBackoff(array $options) + { + $retries = isset($options['retries']) + ? $options['retries'] + : $this->retries; + $retryFunction = isset($options['restRetryFunction']) + ? $options['restRetryFunction'] + : $this->retryFunction; + $delayFunction = isset($options['restDelayFunction']) + ? $options['restDelayFunction'] + : $this->delayFunction; + $backoff = new ExponentialBackoff($retries, $retryFunction); + + if ($delayFunction) { + $backoff->setDelayFunction($delayFunction); + } + + return $backoff; + } } diff --git a/src/Core/RetryDeciderTrait.php b/src/Core/RetryDeciderTrait.php index a04ad5d2ba36..87accace8bc9 100644 --- a/src/Core/RetryDeciderTrait.php +++ b/src/Core/RetryDeciderTrait.php @@ -77,4 +77,20 @@ private function getRetryFunction($shouldRetryMessages = true) return false; }; } + + /** + * @param array $codes + */ + private function setHttpRetryCodes(array $codes) + { + $this->httpRetryCodes = $codes; + } + + /** + * @param array $messages + */ + private function setHttpRetryMessages(array $messages) + { + $this->httpRetryMessages = $messages; + } } diff --git a/tests/snippets/BigQuery/BigQueryClientTest.php b/tests/snippets/BigQuery/BigQueryClientTest.php index b6527511ef23..ff54169a825a 100644 --- a/tests/snippets/BigQuery/BigQueryClientTest.php +++ b/tests/snippets/BigQuery/BigQueryClientTest.php @@ -25,7 +25,9 @@ use Google\Cloud\BigQuery\Connection\ConnectionInterface; use Google\Cloud\BigQuery\Dataset; use Google\Cloud\BigQuery\Job; +use Google\Cloud\BigQuery\QueryJobConfiguration; use Google\Cloud\BigQuery\QueryResults; +use Google\Cloud\BigQuery\ValueMapper; use Google\Cloud\Core\Int64; use Google\Cloud\Core\Iterator\ItemIterator; use Google\Cloud\Dev\Snippet\SnippetTestCase; @@ -36,6 +38,11 @@ */ class BigQueryClientTest extends SnippetTestCase { + const JOB_ID = 'myJobId'; + const PROJECT_ID = 'my-awesome-project'; + const CREATE_DISPOSITION = 'CREATE_NEVER'; + const QUERY_STRING = 'SELECT commit FROM `bigquery-public-data.github_repos.commits` LIMIT 100'; + private $connection; private $client; private $result = [ @@ -63,7 +70,7 @@ class BigQueryClientTest extends SnippetTestCase public function setUp() { $this->connection = $this->prophesize(ConnectionInterface::class); - $this->client = \Google\Cloud\Dev\stub(BigQueryClient::class); + $this->client = \Google\Cloud\Dev\stub(BigQueryTestClient::class); $this->client->___setProperty('connection', $this->connection->reveal()); } @@ -75,12 +82,51 @@ public function testClass() $this->assertInstanceOf(BigQueryClient::class, $res->returnVal()); } + public function testQuery() + { + $snippet = $this->snippetFromMethod(BigQueryClient::class, 'query'); + $snippet->addLocal('bigQuery', $this->client); + $config = $snippet->invoke('queryJobConfig') + ->returnVal(); + + $this->assertInstanceOf(QueryJobConfiguration::class, $config); + $this->assertEquals( + self::QUERY_STRING, + $config->toArray()['configuration']['query']['query'] + ); + } + + public function testQueryWithFluentSetters() + { + $snippet = $this->snippetFromMethod(BigQueryClient::class, 'query', 1); + $snippet->addLocal('bigQuery', $this->client); + $config = $snippet->invoke('queryJobConfig') + ->returnVal(); + $array = $config->toArray(); + + $this->assertInstanceOf(QueryJobConfiguration::class, $config); + $this->assertEquals(self::QUERY_STRING, $array['configuration']['query']['query']); + $this->assertEquals(self::CREATE_DISPOSITION, $array['configuration']['query']['createDisposition']); + } + + public function testQueryWithArrayOfOptions() + { + $snippet = $this->snippetFromMethod(BigQueryClient::class, 'query', 2); + $snippet->addLocal('bigQuery', $this->client); + $config = $snippet->invoke('queryJobConfig') + ->returnVal(); + $array = $config->toArray(); + + $this->assertInstanceOf(QueryJobConfiguration::class, $config); + $this->assertEquals(self::QUERY_STRING, $array['configuration']['query']['query']); + $this->assertEquals(self::CREATE_DISPOSITION, $array['configuration']['query']['createDisposition']); + } + public function testRunQuery() { $snippet = $this->snippetFromMethod(BigQueryClient::class, 'runQuery'); $snippet->addLocal('bigQuery', $this->client); - $snippet->replace('sleep(1);', ''); - $this->connection->query(Argument::any()) + $this->connection->insertJob(Argument::any()) ->shouldBeCalled() ->willReturn([ 'jobComplete' => false, @@ -102,28 +148,40 @@ public function testRunQueryWithNamedParameters() { $snippet = $this->snippetFromMethod(BigQueryClient::class, 'runQuery', 1); $snippet->addLocal('bigQuery', $this->client); - $snippet->replace('sleep(1);', ''); + $expectedQuery = 'SELECT commit FROM `bigquery-public-data.github_repos.commits`' . + 'WHERE author.date < @date AND message = @message LIMIT 100'; $this->connection - ->query(Argument::withEntry('queryParameters', [ - [ - 'name' => 'date', - 'parameterType' => [ - 'type' => 'TIMESTAMP' - ], - 'parameterValue' => [ - 'value' => '1980-01-01 12:15:00.000000+00:00' - ] - ], - [ - 'name' => 'message', - 'parameterType' => [ - 'type' => 'STRING' - ], - 'parameterValue' => [ - 'value' => 'A commit message.' + ->insertJob([ + 'projectId' => self::PROJECT_ID, + 'jobReference' => ['projectId' => self::PROJECT_ID, 'jobId' => self::JOB_ID], + 'configuration' => [ + 'query' => [ + 'parameterMode' => 'named', + 'useLegacySql' => false, + 'queryParameters' => [ + [ + 'name' => 'date', + 'parameterType' => [ + 'type' => 'TIMESTAMP' + ], + 'parameterValue' => [ + 'value' => '1980-01-01 12:15:00.000000+00:00' + ] + ], + [ + 'name' => 'message', + 'parameterType' => [ + 'type' => 'STRING' + ], + 'parameterValue' => [ + 'value' => 'A commit message.' + ] + ] + ], + 'query' => $expectedQuery ] ] - ])) + ]) ->shouldBeCalledTimes(1) ->willReturn([ 'jobComplete' => false, @@ -145,18 +203,29 @@ public function testRunQueryWithPositionalParameters() { $snippet = $this->snippetFromMethod(BigQueryClient::class, 'runQuery', 2); $snippet->addLocal('bigQuery', $this->client); - $snippet->replace('sleep(1);', ''); + $expectedQuery = 'SELECT commit FROM `bigquery-public-data.github_repos.commits` WHERE message = ? LIMIT 100'; $this->connection - ->query(Argument::withEntry('queryParameters', [ - [ - 'parameterType' => [ - 'type' => 'STRING' - ], - 'parameterValue' => [ - 'value' => 'A commit message.' + ->insertJob([ + 'projectId' => self::PROJECT_ID, + 'jobReference' => ['projectId' => self::PROJECT_ID, 'jobId' => self::JOB_ID], + 'configuration' => [ + 'query' => [ + 'parameterMode' => 'positional', + 'useLegacySql' => false, + 'queryParameters' => [ + [ + 'parameterType' => [ + 'type' => 'STRING' + ], + 'parameterValue' => [ + 'value' => 'A commit message.' + ] + ] + ], + 'query' => $expectedQuery ] ] - ])) + ]) ->shouldBeCalledTimes(1) ->willReturn([ 'jobComplete' => false, @@ -174,11 +243,10 @@ public function testRunQueryWithPositionalParameters() $this->assertEquals('abcd', $res->output()); } - public function testRunQueryAsJob() + public function testStartQuery() { - $snippet = $this->snippetFromMethod(BigQueryClient::class, 'runQueryAsJob'); + $snippet = $this->snippetFromMethod(BigQueryClient::class, 'startQuery'); $snippet->addLocal('bigQuery', $this->client); - $snippet->replace('sleep(1);', ''); $this->connection->insertJob(Argument::any()) ->shouldBeCalledTimes(1) ->willReturn([ @@ -332,3 +400,23 @@ public function testTimestamp() $this->assertInstanceOf(Timestamp::class, $res->returnVal()); } } + +class BigQueryTestClient extends BigQueryClient +{ + public function query($query, array $options = []) + { + return (new QueryJobConfigurationStub( + new ValueMapper(false), + BigQueryClientTest::PROJECT_ID, + $options + ))->query($query); + } +} + +class QueryJobConfigurationStub extends QueryJobConfiguration +{ + protected function generateJobId() + { + return BigQueryClientTest::JOB_ID; + } +} diff --git a/tests/snippets/BigQuery/CopyJobConfigurationTest.php b/tests/snippets/BigQuery/CopyJobConfigurationTest.php new file mode 100644 index 000000000000..ddfab1f03a0d --- /dev/null +++ b/tests/snippets/BigQuery/CopyJobConfigurationTest.php @@ -0,0 +1,112 @@ +config = new CopyJobConfiguration( + self::PROJECT_ID, + ['jobReference' => ['jobId' => self::JOB_ID]] + ); + } + + public function testClass() + { + $snippet = $this->snippetFromClass(CopyJobConfiguration::class); + $res = $snippet->invoke('copyJobConfig'); + + $this->assertInstanceOf(CopyJobConfiguration::class, $res->returnVal()); + } + + /** + * @dataProvider setterDataProvider + */ + public function testSetters($method, $expected, $bq = null) + { + $snippet = $this->snippetFromMethod(CopyJobConfiguration::class, $method); + $snippet->addLocal('copyJobConfig', $this->config); + + if ($bq) { + $snippet->addLocal('bigQuery', $bq); + } + + $actual = $snippet->invoke('copyJobConfig') + ->returnVal() + ->toArray()['configuration']['copy'][$method]; + + $this->assertEquals($expected, $actual); + } + + public function setterDataProvider() + { + $bq = \Google\Cloud\Dev\stub(BigQueryClient::class, [ + ['projectId' => self::PROJECT_ID] + ]); + + return [ + [ + 'createDisposition', + 'CREATE_NEVER' + ], + [ + 'destinationEncryptionConfiguration', + [ + 'kmsKeyName' => 'my_key' + ] + ], + [ + 'destinationTable', + [ + 'projectId' => self::PROJECT_ID, + 'datasetId' => self::DATASET_ID, + 'tableId' => self::TABLE_ID + ], + $bq + ], + [ + 'sourceTable', + [ + 'projectId' => self::PROJECT_ID, + 'datasetId' => self::DATASET_ID, + 'tableId' => 'source_table' + ], + $bq + ], + [ + 'writeDisposition', + 'WRITE_TRUNCATE' + ] + ]; + } +} diff --git a/tests/snippets/BigQuery/ExtractJobConfigurationTest.php b/tests/snippets/BigQuery/ExtractJobConfigurationTest.php new file mode 100644 index 000000000000..ae09c5f281f5 --- /dev/null +++ b/tests/snippets/BigQuery/ExtractJobConfigurationTest.php @@ -0,0 +1,110 @@ +config = new ExtractJobConfiguration( + self::PROJECT_ID, + ['jobReference' => ['jobId' => self::JOB_ID]] + ); + } + + public function testClass() + { + $snippet = $this->snippetFromClass(ExtractJobConfiguration::class); + $res = $snippet->invoke('extractJobConfig'); + + $this->assertInstanceOf(ExtractJobConfiguration::class, $res->returnVal()); + } + + /** + * @dataProvider setterDataProvider + */ + public function testSetters($method, $expected, $bq = null) + { + $snippet = $this->snippetFromMethod(ExtractJobConfiguration::class, $method); + $snippet->addLocal('extractJobConfig', $this->config); + + if ($bq) { + $snippet->addLocal('bigQuery', $bq); + } + + $actual = $snippet->invoke('extractJobConfig') + ->returnVal() + ->toArray()['configuration']['extract'][$method]; + + $this->assertEquals($expected, $actual); + } + + public function setterDataProvider() + { + $bq = \Google\Cloud\Dev\stub(BigQueryClient::class, [ + ['projectId' => self::PROJECT_ID] + ]); + + return [ + [ + 'compression', + 'GZIP' + ], + [ + 'destinationFormat', + 'NEWLINE_DELIMITED_JSON' + ], + [ + 'destinationUris', + ['gs://my_bucket/destination.csv'] + ], + [ + 'fieldDelimiter', + ',' + ], + [ + 'printHeader', + false + ], + [ + 'sourceTable', + [ + 'projectId' => self::PROJECT_ID, + 'datasetId' => self::DATASET_ID, + 'tableId' => self::TABLE_ID + ], + $bq + ] + ]; + } +} diff --git a/tests/snippets/BigQuery/JobTest.php b/tests/snippets/BigQuery/JobTest.php index b9a28cbeb2c5..32c6aa58d37c 100644 --- a/tests/snippets/BigQuery/JobTest.php +++ b/tests/snippets/BigQuery/JobTest.php @@ -82,7 +82,6 @@ public function testCancel() ] ]); $snippet = $this->snippetFromMethod(Job::class, 'cancel'); - $snippet->replace('sleep(1);', ''); $snippet->addLocal('job', $job); $snippet->invoke(); } @@ -91,7 +90,7 @@ public function testQueryResults() { $this->connection->getQueryResults(Argument::any()) ->shouldBeCalledTimes(1) - ->willReturn([]); + ->willReturn(['jobComplete' => true]); $job = $this->getJob($this->connection); $snippet = $this->snippetFromMethod(Job::class, 'queryResults'); $snippet->addLocal('job', $job); @@ -100,6 +99,18 @@ public function testQueryResults() $this->assertInstanceOf(QueryResults::class, $res->returnVal()); } + public function testWaitUntilComplete() + { + $job = $this->getJob($this->connection, [ + 'status' => [ + 'state' => 'DONE' + ] + ]); + $snippet = $this->snippetFromMethod(Job::class, 'waitUntilComplete'); + $snippet->addLocal('job', $job); + $snippet->invoke(); + } + public function testIsComplete() { $this->connection->getJob(Argument::any()) @@ -116,7 +127,6 @@ public function testIsComplete() ]); $snippet = $this->snippetFromMethod(Job::class, 'isComplete'); $snippet->addLocal('job', $job); - $snippet->replace('sleep(1);', ''); $snippet->invoke(); } @@ -151,7 +161,6 @@ public function testReload() ]); $snippet = $this->snippetFromMethod(Job::class, 'reload'); $snippet->addLocal('job', $job); - $snippet->replace('sleep(1);', ''); $snippet->invoke(); } diff --git a/tests/snippets/BigQuery/LoadJobConfigurationTest.php b/tests/snippets/BigQuery/LoadJobConfigurationTest.php new file mode 100644 index 000000000000..3a059c9078c5 --- /dev/null +++ b/tests/snippets/BigQuery/LoadJobConfigurationTest.php @@ -0,0 +1,189 @@ +config = new LoadJobConfiguration( + self::PROJECT_ID, + ['jobReference' => ['jobId' => self::JOB_ID]] + ); + } + + public function testClass() + { + $snippet = $this->snippetFromClass(LoadJobConfiguration::class); + $snippet->replace('fopen(\'/path/to/my/data.csv\', \'r\')', '123'); + $res = $snippet->invoke('loadJobConfig'); + + $this->assertInstanceOf(LoadJobConfiguration::class, $res->returnVal()); + } + + public function testData() + { + $snippet = $this->snippetFromMethod(LoadJobConfiguration::class, 'data'); + $snippet->addLocal('loadJobConfig', $this->config); + $snippet->replace('fopen(\'/path/to/my/data.csv\', \'r\')', '123'); + $res = $snippet->invoke('loadJobConfig'); + + $this->assertEquals('123', $res->returnVal()->toArray()['data']); + } + + /** + * @dataProvider setterDataProvider + */ + public function testSetters($method, $expected, $bq = null) + { + $snippet = $this->snippetFromMethod(LoadJobConfiguration::class, $method); + $snippet->addLocal('loadJobConfig', $this->config); + + if ($bq) { + $snippet->addLocal('bigQuery', $bq); + } + + $actual = $snippet->invoke('loadJobConfig') + ->returnVal() + ->toArray()['configuration']['load'][$method]; + + $this->assertEquals($expected, $actual); + } + + public function setterDataProvider() + { + $bq = \Google\Cloud\Dev\stub(BigQueryClient::class, [ + ['projectId' => self::PROJECT_ID] + ]); + + return [ + [ + 'allowJaggedRows', + true + ], + [ + 'allowQuotedNewlines', + true + ], + [ + 'autodetect', + true + ], + [ + 'createDisposition', + 'CREATE_NEVER' + ], + [ + 'destinationEncryptionConfiguration', + [ + 'kmsKeyName' => 'my_key' + ] + ], + [ + 'destinationTable', + [ + 'projectId' => self::PROJECT_ID, + 'datasetId' => self::DATASET_ID, + 'tableId' => self::TABLE_ID + ], + $bq + ], + [ + 'encoding', + 'UTF-8' + ], + [ + 'fieldDelimiter', + '\t' + ], + [ + 'ignoreUnknownValues', + true + ], + [ + 'maxBadRecords', + 10 + ], + [ + 'nullMarker', + '\N', + ], + [ + 'projectionFields', + ['field_name'] + ], + [ + 'quote', + '"' + ], + [ + 'schema', + [ + 'fields' => [ + [ + 'name' => 'col1', + 'type' => 'STRING' + ], + [ + 'name' => 'col2', + 'type' => 'BOOL' + ] + ] + ] + ], + [ + 'schemaUpdateOptions', + ['ALLOW_FIELD_ADDITION'] + ], + [ + 'skipLeadingRows', + 10 + ], + [ + 'sourceFormat', + 'NEWLINE_DELIMITED_JSON' + ], + [ + 'sourceUris', + ['gs://my_bucket/source.csv'] + ], + [ + 'timePartitioning', + ['type' => 'DAY'] + ], + [ + 'writeDisposition', + 'WRITE_TRUNCATE' + ] + ]; + } +} diff --git a/tests/snippets/BigQuery/QueryJobConfigurationTest.php b/tests/snippets/BigQuery/QueryJobConfigurationTest.php new file mode 100644 index 000000000000..eeb100e2adf0 --- /dev/null +++ b/tests/snippets/BigQuery/QueryJobConfigurationTest.php @@ -0,0 +1,166 @@ +config = new QueryJobConfiguration( + new ValueMapper(false), + self::PROJECT_ID, + ['jobReference' => ['jobId' => self::JOB_ID]] + ); + } + + public function testClass() + { + $snippet = $this->snippetFromClass(QueryJobConfiguration::class); + $res = $snippet->invoke('query'); + + $this->assertInstanceOf(QueryJobConfiguration::class, $res->returnVal()); + } + + /** + * @dataProvider setterDataProvider + */ + public function testSetters($method, $expected, $bq = null) + { + $snippet = $this->snippetFromMethod(QueryJobConfiguration::class, $method); + $snippet->addLocal('query', $this->config); + + if ($bq) { + $snippet->addLocal('bigQuery', $bq); + } + + $actual = $snippet->invoke('query') + ->returnVal() + ->toArray()['configuration']['query'][$method]; + + $this->assertEquals($expected, $actual); + } + + public function setterDataProvider() + { + $bq = \Google\Cloud\Dev\stub(BigQueryClient::class, [ + ['projectId' => self::PROJECT_ID] + ]); + + return [ + [ + 'allowLargeResults', + true + ], + [ + 'createDisposition', + 'CREATE_NEVER' + ], + [ + 'defaultDataset', + [ + 'projectId' => self::PROJECT_ID, + 'datasetId' => self::DATASET_ID + ], + $bq + ], + [ + 'destinationEncryptionConfiguration', + [ + 'kmsKeyName' => 'my_key' + ] + ], + [ + 'destinationTable', + [ + 'projectId' => self::PROJECT_ID, + 'datasetId' => self::DATASET_ID, + 'tableId' => self::TABLE_ID + ], + $bq + ], + [ + 'flattenResults', + true + ], + [ + 'maximumBillingTier', + 1 + ], + [ + 'maximumBytesBilled', + 3000 + ], + [ + 'priority', + 'BATCH' + ], + [ + 'query', + 'SELECT commit FROM `bigquery-public-data.github_repos.commits` LIMIT 100', + ], + [ + 'schemaUpdateOptions', + ['ALLOW_FIELD_ADDITION'] + ], + [ + 'tableDefinitions', + [ + 'autodetect' => true, + 'sourceUris' => [ + 'gs://my_bucket/table.json' + ] + ] + ], + [ + 'timePartitioning', + ['type' => 'DAY'] + ], + [ + 'useLegacySql', + true + ], + [ + 'useQueryCache', + true + ], + [ + 'userDefinedFunctionResources', + [['resourceUri' => 'gs://my_bucket/code_path']] + ], + [ + 'writeDisposition', + 'WRITE_TRUNCATE' + ] + ]; + } +} diff --git a/tests/snippets/BigQuery/QueryResultsTest.php b/tests/snippets/BigQuery/QueryResultsTest.php index 058164489466..9efefcbf0d51 100644 --- a/tests/snippets/BigQuery/QueryResultsTest.php +++ b/tests/snippets/BigQuery/QueryResultsTest.php @@ -18,6 +18,7 @@ namespace Google\Cloud\Tests\Snippets\BigQuery; use Google\Cloud\BigQuery\Connection\ConnectionInterface; +use Google\Cloud\BigQuery\Job; use Google\Cloud\BigQuery\QueryResults; use Google\Cloud\BigQuery\ValueMapper; use Google\Cloud\Dev\Snippet\SnippetTestCase; @@ -60,7 +61,6 @@ public function setUp() ] ] ]; - $this->reload = []; $this->connection = $this->prophesize(ConnectionInterface::class); $this->qr = \Google\Cloud\Dev\stub(QueryResults::class, [ @@ -68,8 +68,8 @@ public function setUp() self::JOB_ID, self::PROJECT, $this->info, - $this->reload, - new ValueMapper(false) + new ValueMapper(false), + $this->prophesize(Job::class)->reveal() ]); } @@ -90,6 +90,20 @@ public function testRows() $this->assertEquals('abcd', trim($res->output())); } + public function testWaitUntilComplete() + { + $snippet = $this->snippetFromMethod(QueryResults::class, 'waitUntilComplete'); + $snippet->addLocal('queryResults', $this->qr); + + $this->info['jobComplete'] = true; + $this->connection->getQueryResults(Argument::any()) + ->willReturn($this->info); + + $this->qr->___setProperty('connection', $this->connection->reveal()); + + $snippet->invoke(); + } + public function testIsComplete() { $snippet = $this->snippetFromMethod(QueryResults::class, 'isComplete'); @@ -123,6 +137,15 @@ public function testInfo() $this->assertEquals($this->info['totalBytesProcessed'], $res->output()); } + public function testJob() + { + $snippet = $this->snippetFromMethod(QueryResults::class, 'job'); + $snippet->addLocal('queryResults', $this->qr); + + $res = $snippet->invoke('job'); + $this->assertInstanceOf(Job::class, $res->returnVal()); + } + public function testReload() { $this->connection->getQueryResults(Argument::any()) @@ -133,9 +156,7 @@ public function testReload() $snippet = $this->snippetFromMethod(QueryResults::class, 'reload'); $snippet->addLocal('queryResults', $this->qr); - $snippet->replace('sleep(1);', ''); $res = $snippet->invoke(); - $this->assertEquals('Query complete!', $res->output()); } } diff --git a/tests/snippets/BigQuery/TableTest.php b/tests/snippets/BigQuery/TableTest.php index 9cd00a153362..b20b648ccf1e 100644 --- a/tests/snippets/BigQuery/TableTest.php +++ b/tests/snippets/BigQuery/TableTest.php @@ -19,8 +19,12 @@ use Google\Cloud\BigQuery\BigQueryClient; use Google\Cloud\BigQuery\Connection\ConnectionInterface; +use Google\Cloud\BigQuery\CopyJobConfiguration; +use Google\Cloud\BigQuery\ExtractJobConfiguration; use Google\Cloud\BigQuery\InsertResponse; use Google\Cloud\BigQuery\Job; +use Google\Cloud\BigQuery\JobConfigurationInterface; +use Google\Cloud\BigQuery\LoadJobConfiguration; use Google\Cloud\BigQuery\Table; use Google\Cloud\BigQuery\ValueMapper; use Google\Cloud\Core\Iterator\ItemIterator; @@ -38,6 +42,7 @@ class TableTest extends SnippetTestCase const ID = 'foo'; const DSID = 'bar'; const PROJECT = 'my-awesome-project'; + const JOB_ID = '123'; private $info; private $connection; @@ -137,102 +142,158 @@ public function testRows() $this->assertEquals('abcd' . PHP_EOL, $res->output()); } - public function testCopy() + public function testRunJob() { - $bq = \Google\Cloud\Dev\stub(BigQueryClient::class); - $snippet = $this->snippetFromMethod(Table::class, 'copy'); - $snippet->addLocal('bigQuery', $bq); - - $bq->___setProperty('connection', $this->connection->reveal()); - + $jobConfig = $this->prophesize(JobConfigurationInterface::class); + $jobConfig->toArray() + ->willReturn([ + 'projectId' => self::PROJECT, + 'jobReference' => [ + 'jobId' => self::JOB_ID, + 'projectId' => self::PROJECT + ] + ]); + $snippet = $this->snippetFromMethod(Table::class, 'runJob'); + $snippet->addLocal('table', $this->table); + $snippet->addLocal('jobConfig', $jobConfig->reveal()); $this->connection->insertJob(Argument::any()) ->shouldBeCalled() ->willReturn([ 'jobReference' => [ - 'jobId' => '123' + 'jobId' => self::JOB_ID + ], + 'status' => [ + 'state' => 'DONE' ] ]); - $this->table->___setProperty('connection', $this->connection->reveal()); - $res = $snippet->invoke('job'); + $this->assertInstanceOf(Job::class, $res->returnVal()); + $this->assertEquals('1', $res->output()); } - public function testExport() + public function testStartJob() { - $storage = \Google\Cloud\Dev\stub(StorageClient::class); - $storage->___setProperty('connection', $this->prophesize(StorageConnection::class)->reveal()); - - $snippet = $this->snippetFromMethod(Table::class, 'export'); - $snippet->addLocal('storage', $storage); + $jobConfig = $this->prophesize(JobConfigurationInterface::class); + $jobConfig->toArray() + ->willReturn([ + 'projectId' => self::PROJECT, + 'jobReference' => [ + 'jobId' => self::JOB_ID, + 'projectId' => self::PROJECT + ] + ]); + $snippet = $this->snippetFromMethod(Table::class, 'startJob'); $snippet->addLocal('table', $this->table); - + $snippet->addLocal('jobConfig', $jobConfig->reveal()); $this->connection->insertJob(Argument::any()) ->shouldBeCalled() ->willReturn([ 'jobReference' => [ - 'jobId' => '123' + 'jobId' => self::JOB_ID ] ]); - $this->table->___setProperty('connection', $this->connection->reveal()); - $res = $snippet->invoke('job'); + $this->assertInstanceOf(Job::class, $res->returnVal()); } + public function testCopy() + { + $bq = \Google\Cloud\Dev\stub(BigQueryClient::class); + $snippet = $this->snippetFromMethod(Table::class, 'copy'); + $snippet->addLocal('bigQuery', $bq); + $bq->___setProperty('connection', $this->connection->reveal()); + $config = $snippet->invoke('copyJobConfig') + ->returnVal(); + + $this->assertInstanceOf(CopyJobConfiguration::class, $config); + $this->assertEquals( + [ + 'destinationTable' => [ + 'projectId' => self::PROJECT, + 'datasetId' => 'myDataset', + 'tableId' => 'myDestinationTable' + ], + 'sourceTable' => [ + 'projectId' => self::PROJECT, + 'datasetId' => 'myDataset', + 'tableId' => 'mySourceTable' + ] + ], + $config->toArray()['configuration']['copy'] + ); + } + + public function testExtract() + { + $storage = \Google\Cloud\Dev\stub(StorageClient::class); + $storage->___setProperty('connection', $this->prophesize(StorageConnection::class)->reveal()); + $snippet = $this->snippetFromMethod(Table::class, 'extract'); + $snippet->addLocal('storage', $storage); + $snippet->addLocal('table', $this->table); + $config = $snippet->invoke('extractJobConfig') + ->returnVal(); + + $this->assertInstanceOf(ExtractJobConfiguration::class, $config); + $this->assertEquals( + [ + 'destinationUris' => ['gs://myBucket/tableOutput'], + 'sourceTable' => [ + 'projectId' => self::PROJECT, + 'datasetId' => self::DSID, + 'tableId' => self::ID + ] + ], + $config->toArray()['configuration']['extract'] + ); + } + public function testLoad() { $snippet = $this->snippetFromMethod(Table::class, 'load'); $snippet->addLocal('table', $this->table); - $snippet->replace('/path/to/my/data.csv', 'php://temp'); - - $uploader = $this->prophesize(MultipartUploader::class); - $uploader->upload() - ->shouldBeCalled() - ->willReturn([ - 'jobReference' => [ - 'jobId' => '123' + $snippet->replace('fopen(\'/path/to/my/data.csv\', \'r\')', '123'); + $config = $snippet->invoke('loadJobConfig') + ->returnVal(); + + $this->assertInstanceOf(LoadJobConfiguration::class, $config); + $this->assertEquals( + [ + 'destinationTable' => [ + 'projectId' => self::PROJECT, + 'datasetId' => self::DSID, + 'tableId' => self::ID ] - ]); - - $this->connection->insertJobUpload(Argument::any()) - ->shouldBeCalled() - ->willReturn($uploader->reveal()); - - $this->table->___setProperty('connection', $this->connection->reveal()); - - $res = $snippet->invoke('job'); - $this->assertInstanceOf(Job::class, $res->returnVal()); + ], + $config->toArray()['configuration']['load'] + ); } public function testLoadFromStorage() { $storage = \Google\Cloud\Dev\stub(StorageClient::class); $storage->___setProperty('connection', $this->prophesize(StorageConnection::class)->reveal()); - $snippet = $this->snippetFromMethod(Table::class, 'loadFromStorage'); $snippet->addLocal('storage', $storage); $snippet->addLocal('table', $this->table); - - $uploader = $this->prophesize(MultipartUploader::class); - $uploader->upload() - ->shouldBeCalled() - ->willReturn([ - 'jobReference' => [ - 'jobId' => '123' + $config = $snippet->invoke('loadJobConfig') + ->returnVal(); + + $this->assertInstanceOf(LoadJobConfiguration::class, $config); + $this->assertEquals( + [ + 'sourceUris' => ['gs://myBucket/important-data.csv'], + 'destinationTable' => [ + 'projectId' => self::PROJECT, + 'datasetId' => self::DSID, + 'tableId' => self::ID ] - ]); - - $this->connection->insertJobUpload(Argument::any()) - ->shouldBeCalled() - ->willReturn($uploader->reveal()); - - $this->table->___setProperty('connection', $this->connection->reveal()); - - $res = $snippet->invoke('job'); - $this->assertInstanceOf(Job::class, $res->returnVal()); + ], + $config->toArray()['configuration']['load'] + ); } public function testInsertRow() diff --git a/tests/system/BigQuery/LoadDataAndQueryTest.php b/tests/system/BigQuery/LoadDataAndQueryTest.php index 953f64f69499..e0ebf71001a4 100644 --- a/tests/system/BigQuery/LoadDataAndQueryTest.php +++ b/tests/system/BigQuery/LoadDataAndQueryTest.php @@ -91,16 +91,16 @@ public function testInsertRowToTable() */ public function testRunQuery($useLegacySql) { - $query = sprintf( + $queryString = sprintf( $useLegacySql ? 'SELECT Name, Age, Weight, IsMagic, Spells.* FROM [%s.%s]' : 'SELECT Name, Age, Weight, IsMagic, Spells FROM `%s.%s`', self::$dataset->id(), self::$table->id() ); - $results = self::$client->runQuery($query, [ - 'useLegacySql' => $useLegacySql - ]); + $query = self::$client->query($queryString) + ->useLegacySql($useLegacySql); + $results = self::$client->runQuery($query); $backoff = new ExponentialBackoff(8); $backoff->execute(function () use ($results) { $results->reload(); @@ -147,20 +147,18 @@ public function testRunQuery($useLegacySql) * @depends testInsertRowToTable * @dataProvider useLegacySqlProvider */ - public function testRunQueryAsJob($useLegacySql) + public function testStartQuery($useLegacySql) { - $query = sprintf( + $queryString = sprintf( $useLegacySql ? 'SELECT FavoriteNumbers, ImportantDates.* FROM [%s.%s]' : 'SELECT FavoriteNumbers, ImportantDates FROM `%s.%s`', self::$dataset->id(), self::$table->id() ); - $job = self::$client->runQueryAsJob($query, [ - 'jobConfig' => [ - 'useLegacySql' => $useLegacySql - ] - ]); + $query = self::$client->query($queryString) + ->useLegacySql($useLegacySql); + $job = self::$client->startQuery($query); $results = $job->queryResults(); $backoff = new ExponentialBackoff(8); $backoff->execute(function () use ($results) { @@ -206,7 +204,7 @@ public function useLegacySqlProvider() public function testRunQueryWithNamedParameters() { - $query = 'SELECT' + $queryString = 'SELECT' . '@structType as structType,' . '@arrayStruct as arrayStruct,' . '@nestedStruct as nestedStruct,' @@ -245,7 +243,9 @@ public function testRunQueryWithNamedParameters() 'time' => self::$client->time(new \DateTime('11:15:02')), 'bytes' => $bytes ]; - $results = self::$client->runQuery($query, ['parameters' => $params]); + $query = self::$client->query($queryString) + ->parameters($params); + $results = self::$client->runQuery($query); $backoff = new ExponentialBackoff(8); $backoff->execute(function () use ($results) { @@ -271,19 +271,11 @@ public function testRunQueryWithNamedParameters() public function testRunQueryWithPositionalParameters() { - $results = self::$client->runQuery('SELECT 1 IN UNNEST(?) AS arr', [ - 'parameters' => [ + $query = self::$client->query('SELECT 1 IN UNNEST(?) AS arr') + ->parameters([ [1, 2, 3] - ] - ]); - $backoff = new ExponentialBackoff(8); - $backoff->execute(function () use ($results) { - $results->reload(); - - if (!$results->isComplete()) { - throw new \Exception(); - } - }); + ]); + $results = self::$client->runQuery($query); if (!$results->isComplete()) { $this->fail('Query did not complete within the allotted time.'); @@ -297,14 +289,13 @@ public function testRunQueryWithPositionalParameters() $this->assertEquals($expectedRows, $actualRows); } - public function testRunQueryAsJobWithNamedParameters() + public function testStartQueryWithNamedParameters() { - $query = 'SELECT @int as int'; - $job = self::$client->runQueryAsJob($query, [ - 'parameters' => [ + $query = self::$client->query('SELECT @int as int') + ->parameters([ 'int' => 5 - ] - ]); + ]); + $job = self::$client->startQuery($query); $results = $job->queryResults(); $backoff = new ExponentialBackoff(8); $backoff->execute(function () use ($results) { @@ -325,13 +316,13 @@ public function testRunQueryAsJobWithNamedParameters() $this->assertEquals($expectedRows, $actualRows); } - public function testRunQueryAsJobWithPositionalParameters() + public function testStartQueryWithPositionalParameters() { - $job = self::$client->runQueryAsJob('SELECT 1 IN UNNEST(?) AS arr', [ - 'parameters' => [ + $query = self::$client->query('SELECT 1 IN UNNEST(?) AS arr') + ->parameters([ [1, 2, 3] - ] - ]); + ]); + $job = self::$client->startQuery($query); $results = $job->queryResults(); $backoff = new ExponentialBackoff(8); $backoff->execute(function () use ($results) { @@ -356,15 +347,13 @@ public function testRunQueryAsJobWithPositionalParameters() /** * @dataProvider rowProvider - * @depends testInsertRowToTable */ public function testLoadsDataToTable($data) { - $job = self::$table->load($data, [ - 'jobConfig' => [ - 'sourceFormat' => 'NEWLINE_DELIMITED_JSON' - ] - ]); + $loadJobConfig = self::$table->load($data) + ->sourceFormat('NEWLINE_DELIMITED_JSON'); + + $job = self::$table->startJob($loadJobConfig); $backoff = new ExponentialBackoff(8); $backoff->execute(function () use ($job) { $job->reload(); @@ -404,11 +393,9 @@ public function testLoadsDataFromStorageToTable() fopen(__DIR__ . '/../data/table-data.json', 'r') ); - $job = self::$table->loadFromStorage($object, [ - 'jobConfig' => [ - 'sourceFormat' => 'NEWLINE_DELIMITED_JSON' - ] - ]); + $loadJobConfig = self::$table->loadFromStorage($object) + ->sourceFormat('NEWLINE_DELIMITED_JSON'); + $job = self::$table->startJob($loadJobConfig); $backoff = new ExponentialBackoff(8); $backoff->execute(function () use ($job) { $job->reload(); @@ -443,4 +430,33 @@ public function testInsertRowsToTable() $this->assertTrue($insertResponse->isSuccessful()); $this->assertEquals(self::$expectedRows, $actualRows); } + + public function testInsertRowsToTableWithAutoCreate() + { + $tName = uniqid(BigQueryTestCase::TESTING_PREFIX); + $rows = [ + ['data' => ['hello' => 'world']] + ]; + $insertResponse = self::$dataset->table($tName) + ->insertRows($rows, [ + 'autoCreate' => true, + 'tableMetadata' => [ + 'schema' => [ + 'fields' => [ + [ + 'name' => 'hello', + 'type' => 'STRING' + ] + ] + ] + ] + ]); + $results = self::$dataset + ->table($tName) + ->rows(); + $actualRows = count(iterator_to_array($results)); + + $this->assertTrue($insertResponse->isSuccessful()); + $this->assertEquals(count($rows), $actualRows); + } } diff --git a/tests/system/BigQuery/ManageDatasetsTest.php b/tests/system/BigQuery/ManageDatasetsTest.php index bb797dce33c2..f6b1ffe1276b 100644 --- a/tests/system/BigQuery/ManageDatasetsTest.php +++ b/tests/system/BigQuery/ManageDatasetsTest.php @@ -75,6 +75,19 @@ public function testUpdateDataset() $this->assertEquals($metadata['friendlyName'], $info['friendlyName']); } + /** + * @expectedException Google\Cloud\Core\Exception\FailedPreconditionException + */ + public function testUpdateDatasetConcurrentUpdateFails() + { + $data = [ + 'friendlyName' => 'foo', + 'etag' => 'blah' + ]; + + self::$dataset->update($data); + } + public function testReloadsDataset() { $this->assertEquals('bigquery#dataset', self::$dataset->reload()['kind']); diff --git a/tests/system/BigQuery/ManageJobsTest.php b/tests/system/BigQuery/ManageJobsTest.php index b6ecaad270be..bd4622694690 100644 --- a/tests/system/BigQuery/ManageJobsTest.php +++ b/tests/system/BigQuery/ManageJobsTest.php @@ -27,13 +27,12 @@ class ManageJobsTest extends BigQueryTestCase { public function testListJobs() { - self::$client->runQueryAsJob( - sprintf( - 'SELECT * FROM [%s.%s]', - self::$dataset->id(), - self::$table->id() - ) - ); + $query = self::$client->query(sprintf( + 'SELECT * FROM [%s.%s]', + self::$dataset->id(), + self::$table->id() + )); + self::$client->startQuery($query); $jobs = self::$client->jobs(); $job = null; @@ -59,13 +58,12 @@ public function testJobExists($job) public function testCancelsJob() { - $job = self::$client->runQueryAsJob( - sprintf( - 'SELECT * FROM [%s.%s]', - self::$dataset->id(), - self::$table->id() - ) - ); + $query = self::$client->query(sprintf( + 'SELECT * FROM [%s.%s]', + self::$dataset->id(), + self::$table->id() + )); + $job = self::$client->startQuery($query); $job->cancel(); } diff --git a/tests/system/BigQuery/ManageTablesTest.php b/tests/system/BigQuery/ManageTablesTest.php index 5d9a40adfd6b..4e53cbc466ca 100644 --- a/tests/system/BigQuery/ManageTablesTest.php +++ b/tests/system/BigQuery/ManageTablesTest.php @@ -74,7 +74,8 @@ public function testCreatesTable() */ public function testCopiesTable($table) { - $job = self::$table->copy($table); + $copyJobConfig = self::$table->copy($table); + $job = self::$table->startJob($copyJobConfig); $backoff = new ExponentialBackoff(8); $backoff->execute(function () use ($job) { $job->reload(); @@ -91,17 +92,15 @@ public function testCopiesTable($table) $this->assertArrayNotHasKey('errorResult', $job->info()['status']); } - public function testExportsTable() + public function testExtractsTable() { $object = self::$bucket->object( uniqid(self::TESTING_PREFIX) ); - $job = self::$table->export($object, [ - 'jobConfig' => [ - 'destinationFormat' => 'NEWLINE_DELIMITED_JSON' - ] - ]); + $extractJobConfig = self::$table->extract($object) + ->destinationFormat('NEWLINE_DELIMITED_JSON'); + $job = self::$table->startJob($extractJobConfig); $backoff = new ExponentialBackoff(8); $backoff->execute(function () use ($job) { @@ -129,6 +128,19 @@ public function testUpdateTable() $this->assertEquals($metadata['friendlyName'], $info['friendlyName']); } + /** + * @expectedException Google\Cloud\Core\Exception\FailedPreconditionException + */ + public function testUpdateTableConcurrentUpdateFails() + { + $data = [ + 'friendlyName' => 'foo', + 'etag' => 'blah' + ]; + + self::$table->update($data); + } + public function testReloadsTable() { $this->assertEquals('bigquery#table', self::$table->reload()['kind']); diff --git a/tests/unit/BigQuery/BigQueryClientTest.php b/tests/unit/BigQuery/BigQueryClientTest.php index 190691fffd9a..349a14e9f2fc 100644 --- a/tests/unit/BigQuery/BigQueryClientTest.php +++ b/tests/unit/BigQuery/BigQueryClientTest.php @@ -23,9 +23,11 @@ use Google\Cloud\BigQuery\Dataset; use Google\Cloud\BigQuery\Date; use Google\Cloud\BigQuery\Job; +use Google\Cloud\BigQuery\QueryJobConfiguration; use Google\Cloud\BigQuery\QueryResults; use Google\Cloud\BigQuery\Time; use Google\Cloud\BigQuery\Timestamp; +use Google\Cloud\Core\Iterator\ItemIterator; use Prophecy\Argument; /** @@ -33,145 +35,141 @@ */ class BigQueryClientTest extends \PHPUnit_Framework_TestCase { + const JOB_ID = 'myJobId'; + const PROJECT_ID = 'myProjectId'; + const DATASET_ID = 'myDatasetId'; + const QUERY_STRING = 'someQuery'; + public $connection; - public $jobId = 'myJobId'; - public $projectId = 'myProjectId'; - public $datasetId = 'myDatasetId'; public $client; public function setUp() { $this->connection = $this->prophesize(ConnectionInterface::class); - $this->client = new BigQueryTestClient(['projectId' => $this->projectId]); + $this->client = \Google\Cloud\Dev\stub(BigQueryClient::class, ['options' => ['projectId' => self::PROJECT_ID]]); } - /** - * @dataProvider queryDataProvider - */ - public function testRunsQuery($query, $options, $expected) + public function testRunsQuery() { - $this->connection->query($expected) + $query = $this->client->query(self::QUERY_STRING, [ + 'jobReference' => ['jobId' => self::JOB_ID] + ]); + $this->connection->insertJob([ + 'projectId' => self::PROJECT_ID, + 'configuration' => [ + 'query' => [ + 'query' => self::QUERY_STRING, + 'useLegacySql' => false + ] + ], + 'jobReference' => [ + 'projectId' => self::PROJECT_ID, + 'jobId' => self::JOB_ID + ] + ]) + ->willReturn([ + 'jobReference' => ['jobId' => self::JOB_ID] + ]) + ->shouldBeCalledTimes(1); + $this->connection->getQueryResults(Argument::any()) ->willReturn([ 'jobReference' => [ - 'jobId' => $this->jobId - ] + 'jobId' => self::JOB_ID + ], + 'jobComplete' => true ]) ->shouldBeCalledTimes(1); - $this->client->setConnection($this->connection->reveal()); - $queryResults = $this->client->runQuery($query, $options); + $this->client->___setProperty('connection', $this->connection->reveal()); + $queryResults = $this->client->runQuery($query); $this->assertInstanceOf(QueryResults::class, $queryResults); - $this->assertEquals($this->jobId, $queryResults->identity()['jobId']); + $this->assertEquals(self::JOB_ID, $queryResults->identity()['jobId']); } - /** - * @dataProvider queryDataProvider - */ - public function testRunsQueryAsJob($query, $options, $expected) + public function testRunsQueryWithRetry() { - $projectId = $expected['projectId']; - unset($expected['projectId']); + $query = $this->client->query(self::QUERY_STRING, [ + 'jobReference' => ['jobId' => self::JOB_ID] + ]); $this->connection->insertJob([ - 'projectId' => $projectId, + 'projectId' => self::PROJECT_ID, 'configuration' => [ - 'query' => $expected + 'query' => [ + 'query' => self::QUERY_STRING, + 'useLegacySql' => false + ] + ], + 'jobReference' => [ + 'projectId' => self::PROJECT_ID, + 'jobId' => self::JOB_ID ] ]) ->willReturn([ - 'jobReference' => ['jobId' => $this->jobId] + 'jobReference' => [ + 'jobId' => self::JOB_ID + ], + 'jobComplete' => false + ]) + ->shouldBeCalledTimes(1); + $this->connection->getQueryResults(Argument::any()) + ->willReturn([ + 'jobReference' => [ + 'jobId' => self::JOB_ID + ], + 'jobComplete' => true ]) ->shouldBeCalledTimes(1); - $this->client->setConnection($this->connection->reveal()); - $job = $this->client->runQueryAsJob($query, $options); - $this->assertInstanceOf(Job::class, $job); - $this->assertEquals($this->jobId, $job->id()); + $this->client->___setProperty('connection', $this->connection->reveal()); + $queryResults = $this->client->runQuery($query); + + $this->assertInstanceOf(QueryResults::class, $queryResults); + $this->assertEquals(self::JOB_ID, $queryResults->identity()['jobId']); } - public function queryDataProvider() + public function testStartQuery() { - $query = 'someQuery'; - - return [ - [ - $query, - [], - [ - 'projectId' => $this->projectId, - 'query' => $query - ] - ], - [ - $query, - [ - 'parameters' => [ - 'test' => 'parameter' - ] - ], - [ - 'projectId' => $this->projectId, - 'query' => $query, - 'parameterMode' => 'named', - 'useLegacySql' => false, - 'queryParameters' => [ - [ - 'name' => 'test', - 'parameterType' => [ - 'type' => 'STRING' - ], - 'parameterValue' => [ - 'value' => 'parameter' - ] - ] - ] + $query = $this->client->query(self::QUERY_STRING, [ + 'jobReference' => ['jobId' => self::JOB_ID] + ]); + $this->connection->insertJob([ + 'projectId' => self::PROJECT_ID, + 'configuration' => [ + 'query' => [ + 'query' => self::QUERY_STRING, + 'useLegacySql' => false ] ], - [ - $query, - [ - 'parameters' => [1, 2] - ], - [ - 'projectId' => $this->projectId, - 'query' => 'someQuery', - 'parameterMode' => 'positional', - 'useLegacySql' => false, - 'queryParameters' => [ - [ - 'parameterType' => [ - 'type' => 'INT64' - ], - 'parameterValue' => [ - 'value' => 1 - ] - ], - [ - 'parameterType' => [ - 'type' => 'INT64' - ], - 'parameterValue' => [ - 'value' => 2 - ] - ] - ] - ] + 'jobReference' => [ + 'projectId' => self::PROJECT_ID, + 'jobId' => self::JOB_ID ] - ]; + ]) + ->willReturn([ + 'jobReference' => ['jobId' => self::JOB_ID] + ]) + ->shouldBeCalledTimes(1); + + $this->client->___setProperty('connection', $this->connection->reveal()); + $job = $this->client->startQuery($query); + + $this->assertInstanceOf(Job::class, $job); + $this->assertEquals(self::JOB_ID, $job->id()); } public function testGetsJob() { - $this->client->setConnection($this->connection->reveal()); - $this->assertInstanceOf(Job::class, $this->client->job($this->jobId)); + $this->client->___setProperty('connection', $this->connection->reveal()); + $this->assertInstanceOf(Job::class, $this->client->job(self::JOB_ID)); } public function testGetsJobsWithNoResults() { - $this->connection->listJobs(['projectId' => $this->projectId]) + $this->connection->listJobs(['projectId' => self::PROJECT_ID]) ->willReturn([]) ->shouldBeCalledTimes(1); - $this->client->setConnection($this->connection->reveal()); + $this->client->___setProperty('connection', $this->connection->reveal()); $jobs = iterator_to_array($this->client->jobs()); $this->assertEmpty($jobs); @@ -179,24 +177,24 @@ public function testGetsJobsWithNoResults() public function testGetsJobsWithoutToken() { - $this->connection->listJobs(['projectId' => $this->projectId]) + $this->connection->listJobs(['projectId' => self::PROJECT_ID]) ->willReturn([ 'jobs' => [ - ['jobReference' => ['jobId' => $this->jobId]] + ['jobReference' => ['jobId' => self::JOB_ID]] ] ]) ->shouldBeCalledTimes(1); - $this->client->setConnection($this->connection->reveal()); + $this->client->___setProperty('connection', $this->connection->reveal()); $jobs = iterator_to_array($this->client->jobs()); - $this->assertEquals($this->jobId, $jobs[0]->id()); + $this->assertEquals(self::JOB_ID, $jobs[0]->id()); } public function testGetsJobsWithToken() { $token = 'token'; - $this->connection->listJobs(['projectId' => $this->projectId]) + $this->connection->listJobs(['projectId' => self::PROJECT_ID]) ->willReturn([ 'nextPageToken' => $token, 'jobs' => [ @@ -204,25 +202,25 @@ public function testGetsJobsWithToken() ] ])->shouldBeCalledTimes(1); $this->connection->listJobs([ - 'projectId' => $this->projectId, + 'projectId' => self::PROJECT_ID, 'pageToken' => $token ]) ->willReturn([ 'jobs' => [ - ['jobReference' => ['jobId' => $this->jobId]] + ['jobReference' => ['jobId' => self::JOB_ID]] ] ])->shouldBeCalledTimes(1); - $this->client->setConnection($this->connection->reveal()); + $this->client->___setProperty('connection', $this->connection->reveal()); $job = iterator_to_array($this->client->jobs()); - $this->assertEquals($this->jobId, $job[1]->id()); + $this->assertEquals(self::JOB_ID, $job[1]->id()); } public function testGetsDataset() { - $this->client->setConnection($this->connection->reveal()); - $this->assertInstanceOf(Dataset::class, $this->client->dataset($this->datasetId)); + $this->client->___setProperty('connection', $this->connection->reveal()); + $this->assertInstanceOf(Dataset::class, $this->client->dataset(self::DATASET_ID)); } public function testGetsDatasetsWithNoResults() @@ -231,7 +229,7 @@ public function testGetsDatasetsWithNoResults() ->willReturn([]) ->shouldBeCalledTimes(1); - $this->client->setConnection($this->connection->reveal()); + $this->client->___setProperty('connection', $this->connection->reveal()); $datasets = iterator_to_array($this->client->datasets()); $this->assertEmpty($datasets); @@ -242,15 +240,15 @@ public function testGetsDatasetsWithoutToken() $this->connection->listDatasets(Argument::any()) ->willReturn([ 'datasets' => [ - ['datasetReference' => ['datasetId' => $this->datasetId]] + ['datasetReference' => ['datasetId' => self::DATASET_ID]] ] ]) ->shouldBeCalledTimes(1); - $this->client->setConnection($this->connection->reveal()); + $this->client->___setProperty('connection', $this->connection->reveal()); $datasets = iterator_to_array($this->client->datasets()); - $this->assertEquals($this->datasetId, $datasets[0]->id()); + $this->assertEquals(self::DATASET_ID, $datasets[0]->id()); } public function testGetsDatasetsWithToken() @@ -265,16 +263,16 @@ public function testGetsDatasetsWithToken() ], [ 'datasets' => [ - ['datasetReference' => ['datasetId' => $this->datasetId]] + ['datasetReference' => ['datasetId' => self::DATASET_ID]] ] ] ) ->shouldBeCalledTimes(2); - $this->client->setConnection($this->connection->reveal()); + $this->client->___setProperty('connection', $this->connection->reveal()); $dataset = iterator_to_array($this->client->datasets()); - $this->assertEquals($this->datasetId, $dataset[1]->id()); + $this->assertEquals(self::DATASET_ID, $dataset[1]->id()); } public function testCreatesDataset() @@ -282,13 +280,13 @@ public function testCreatesDataset() $this->connection->insertDataset(Argument::any()) ->willReturn([ 'datasetReference' => [ - 'datasetId' => $this->datasetId + 'datasetId' => self::DATASET_ID ] ]) ->shouldBeCalledTimes(1); - $this->client->setConnection($this->connection->reveal()); + $this->client->___setProperty('connection', $this->connection->reveal()); - $dataset = $this->client->createDataset($this->datasetId, [ + $dataset = $this->client->createDataset(self::DATASET_ID, [ 'metadata' => [ 'friendlyName' => 'A dataset.' ] @@ -325,11 +323,3 @@ public function testGetsTimestamp() $this->assertInstanceOf(Timestamp::class, $bytes); } } - -class BigQueryTestClient extends BigQueryClient -{ - public function setConnection($connection) - { - $this->connection = $connection; - } -} diff --git a/tests/unit/BigQuery/Connection/RestTest.php b/tests/unit/BigQuery/Connection/RestTest.php index af3a26b5bcad..b23a5ad20e59 100644 --- a/tests/unit/BigQuery/Connection/RestTest.php +++ b/tests/unit/BigQuery/Connection/RestTest.php @@ -98,6 +98,9 @@ public function testInsertJobUpload() { $actualRequest = null; $config = [ + 'labels' => [], + 'dryRun' => false, + 'jobReference' => [], 'configuration' => [ 'load' => [ 'destinationTable' => [ diff --git a/tests/unit/BigQuery/CopyJobConfigurationTest.php b/tests/unit/BigQuery/CopyJobConfigurationTest.php new file mode 100644 index 000000000000..adf21b6dcc86 --- /dev/null +++ b/tests/unit/BigQuery/CopyJobConfigurationTest.php @@ -0,0 +1,91 @@ + self::PROJECT_ID, + 'datasetId' => self::DATASET_ID, + 'tableId' => self::TABLE_ID + ]; + private $expectedConfig; + + public function setUp() + { + $this->expectedConfig = [ + 'projectId' => self::PROJECT_ID, + 'jobReference' => [ + 'projectId' => self::PROJECT_ID, + 'jobId' => self::JOB_ID + ], + 'configuration' => [ + 'copy' => [] + ] + ]; + $this->config = new CopyJobConfiguration( + self::PROJECT_ID, + ['jobReference' => ['jobId' => self::JOB_ID]] + ); + } + + public function testFluentSetters() + { + $destinationTable = $this->prophesize(Table::class); + $destinationTable->identity() + ->willReturn($this->tableIdentity); + $sourceTable = $this->prophesize(Table::class); + $sourceTable->identity() + ->willReturn($this->tableIdentity); + $copy = [ + 'createDisposition' => 'CREATE_NEVER', + 'destinationEncryptionConfiguration' => [ + 'kmsKeyName' => 'my_key' + ], + 'destinationTable' => $this->tableIdentity, + 'sourceTable' => $this->tableIdentity, + 'writeDisposition' => 'WRITE_TRUNCATE' + ]; + $this->expectedConfig['configuration']['copy'] = $copy + + $this->expectedConfig['configuration']['copy']; + $this->config + ->createDisposition($copy['createDisposition']) + ->destinationEncryptionConfiguration($copy['destinationEncryptionConfiguration']) + ->destinationTable($destinationTable->reveal()) + ->sourceTable($sourceTable->reveal()) + ->writeDisposition($copy['writeDisposition']); + + $this->assertEquals( + $this->expectedConfig, + $this->config->toArray() + ); + } +} diff --git a/tests/unit/BigQuery/DatasetTest.php b/tests/unit/BigQuery/DatasetTest.php index 1f9703811f96..678fa95b6109 100644 --- a/tests/unit/BigQuery/DatasetTest.php +++ b/tests/unit/BigQuery/DatasetTest.php @@ -94,6 +94,22 @@ public function testUpdatesData() $this->assertEquals($updateData['friendlyName'], $dataset->info()['friendlyName']); } + public function testUpdatesDataWithEtag() + { + $updateData = ['friendlyName' => 'wow a name', 'etag' => 'foo']; + $this->connection->patchDataset(Argument::that(function ($args) { + if ($args['restOptions']['headers']['If-Match'] !== 'foo') return false; + + return true; + })) + ->willReturn($updateData) + ->shouldBeCalledTimes(1); + $dataset = $this->getDataset($this->connection, ['friendlyName' => 'another name']); + $dataset->update($updateData); + + $this->assertEquals($updateData['friendlyName'], $dataset->info()['friendlyName']); + } + public function testGetsTable() { $dataset = $this->getDataset($this->connection); diff --git a/tests/unit/BigQuery/Exception/JobExceptionTest.php b/tests/unit/BigQuery/Exception/JobExceptionTest.php new file mode 100644 index 000000000000..604918e45657 --- /dev/null +++ b/tests/unit/BigQuery/Exception/JobExceptionTest.php @@ -0,0 +1,39 @@ +prophesize(Job::class)->reveal() + ); + + $this->assertEquals($message, $ex->getMessage()); + $this->assertInstanceOf(Job::class, $ex->getJob()); + } +} diff --git a/tests/unit/BigQuery/ExtractJobConfigurationTest.php b/tests/unit/BigQuery/ExtractJobConfigurationTest.php new file mode 100644 index 000000000000..11cfdc01fc58 --- /dev/null +++ b/tests/unit/BigQuery/ExtractJobConfigurationTest.php @@ -0,0 +1,88 @@ + self::PROJECT_ID, + 'datasetId' => self::DATASET_ID, + 'tableId' => self::TABLE_ID + ]; + private $expectedConfig; + + public function setUp() + { + $this->expectedConfig = [ + 'projectId' => self::PROJECT_ID, + 'jobReference' => [ + 'projectId' => self::PROJECT_ID, + 'jobId' => self::JOB_ID + ], + 'configuration' => [ + 'extract' => [] + ] + ]; + $this->config = new ExtractJobConfiguration( + self::PROJECT_ID, + ['jobReference' => ['jobId' => self::JOB_ID]] + ); + } + + public function testFluentSetters() + { + $sourceTable = $this->prophesize(Table::class); + $sourceTable->identity() + ->willReturn($this->tableIdentity); + $extract = [ + 'compression' => 'GZIP', + 'destinationFormat' => 'CSV', + 'destinationUris' => ['gs://my_bucket/destination.csv'], + 'fieldDelimiter' => ',', + 'printHeader' => true, + 'sourceTable' => $this->tableIdentity + ]; + $this->expectedConfig['configuration']['extract'] = $extract + + $this->expectedConfig['configuration']['extract']; + $this->config + ->compression($extract['compression']) + ->destinationFormat($extract['destinationFormat']) + ->destinationUris($extract['destinationUris']) + ->fieldDelimiter($extract['fieldDelimiter']) + ->printHeader($extract['printHeader']) + ->sourceTable($sourceTable->reveal()); + + $this->assertEquals( + $this->expectedConfig, + $this->config->toArray() + ); + } +} diff --git a/tests/unit/BigQuery/JobConfigurationTraitTest.php b/tests/unit/BigQuery/JobConfigurationTraitTest.php new file mode 100644 index 000000000000..a474179b9936 --- /dev/null +++ b/tests/unit/BigQuery/JobConfigurationTraitTest.php @@ -0,0 +1,109 @@ +trait = \Google\Cloud\Dev\impl(JobConfigurationTrait::class); + } + + public function testJobConfigurationProperties() + { + $this->trait->call('jobConfigurationProperties', [ + self::PROJECT_ID, + ['jobReference' => ['jobId' => self::JOB_ID]] + ]); + + $this->assertEquals([ + 'projectId' => self::PROJECT_ID, + 'jobReference' => [ + 'jobId' => self::JOB_ID, + 'projectId' => self::PROJECT_ID + ] + ], $this->trait->call('toArray')); + } + + public function testJobConfigurationPropertiesSetsJobIDWhenNotProvided() + { + $this->trait->call('jobConfigurationProperties', [ + self::PROJECT_ID, + [] + ]); + $jobId = $this->trait->call('toArray')['jobReference']['jobId']; + + $this->assertTrue(is_string($jobId)); + $this->assertTrue(Uuid::isValid($jobId)); + } + + public function testDryRun() + { + $isDryRun = true; + $this->trait->call('dryRun', [$isDryRun]); + + $this->assertEquals( + $isDryRun, + $this->trait->call('toArray')['configuration']['dryRun'] + ); + } + + public function testJobIdPrefix() + { + $jobIdPrefix = 'prefix'; + $this->trait->call('jobConfigurationProperties', [ + self::PROJECT_ID, + ['jobReference' => ['jobId' => self::JOB_ID]] + ]); + $this->trait->call('jobIdPrefix', [$jobIdPrefix]); + + $this->assertEquals( + sprintf('%s-%s', $jobIdPrefix, self::JOB_ID), + $this->trait->call('toArray')['jobReference']['jobId'] + ); + } + + public function testLabels() + { + $labels = ['test' => 'label']; + $this->trait->call('labels', [$labels]); + + $this->assertEquals( + $labels, + $this->trait->call('toArray')['configuration']['labels'] + ); + } + + public function testGenerateJobId() + { + $uuid = $this->trait->call('generateJobId'); + $this->assertTrue(is_string($uuid)); + $this->assertTrue(Uuid::isValid($uuid)); + } +} diff --git a/tests/unit/BigQuery/JobTest.php b/tests/unit/BigQuery/JobTest.php index b89885d36ef7..b5d51d31856e 100644 --- a/tests/unit/BigQuery/JobTest.php +++ b/tests/unit/BigQuery/JobTest.php @@ -79,13 +79,31 @@ public function testCancel() public function testGetsQueryResults() { $this->connection->getQueryResults(Argument::any()) - ->willReturn(['jobReference' => ['jobId' => $this->jobId]]) + ->willReturn([ + 'jobReference' => [ + 'jobId' => $this->jobId + ], + 'jobComplete' => true + ]) ->shouldBeCalledTimes(1); $job = $this->getJob($this->connection); $this->assertInstanceOf(QueryResults::class, $job->queryResults()); } + public function testWaitsUntilComplete() + { + $this->jobInfo['status']['state'] = 'RUNNING'; + $this->connection->getJob(Argument::any()) + ->willReturn([ + 'status' => [ + 'state' => 'DONE' + ] + ])->shouldBeCalledTimes(1); + $job = $this->getJob($this->connection, $this->jobInfo); + $job->waitUntilComplete(); + } + public function testIsCompleteTrue() { $job = $this->getJob($this->connection, $this->jobInfo); diff --git a/tests/unit/BigQuery/JobWaitTraitTest.php b/tests/unit/BigQuery/JobWaitTraitTest.php new file mode 100644 index 000000000000..3c9d8a7c3ad7 --- /dev/null +++ b/tests/unit/BigQuery/JobWaitTraitTest.php @@ -0,0 +1,94 @@ +trait = \Google\Cloud\Dev\impl(JobWaitTrait::class); + $this->job = $this->prophesize(Job::class)->reveal(); + } + + public function testWaitSucceedsWhenAlreadyComplete() + { + $isCompleteCalled = false; + $isReloadCalled = false; + + $this->trait->call('wait', [ + function() use (&$isCompleteCalled) { + $isCompleteCalled = true; + return true; + }, + function() use (&$isReloadCalled) { + $isReloadCalled = true; + return ['complete' => true]; + }, + $this->job, + 1 + ]); + + $this->assertTrue($isCompleteCalled); + $this->assertFalse($isReloadCalled); + } + + public function testWaitCallsReloadThenSucceeds() + { + $isCompleteCallCount = 0; + $isReloadCalled = false; + + $this->trait->call('wait', [ + function() use (&$isCompleteCallCount, &$isReloadCalled) { + $isCompleteCallCount++; + return $isReloadCalled ? true : false; + }, + function() use (&$isReloadCalled) { + $isReloadCalled = true; + return ['complete' => true]; + }, + $this->job, + 1 + ]); + + $this->assertEquals(2, $isCompleteCallCount); + $this->assertTrue($isReloadCalled); + } + + /** + * @expectedException Google\Cloud\BigQuery\Exception\JobException + * @expectedExceptionMessage Job did not complete within the allowed number of retries. + */ + public function testWaitThrowsExceptionWhenMaxAttemptsMet() + { + $this->trait->call('wait', [ + function() { return false; }, + function () { return ['complete' => false]; }, + $this->job, + 1 + ]); + } +} diff --git a/tests/unit/BigQuery/LoadJobConfigurationTest.php b/tests/unit/BigQuery/LoadJobConfigurationTest.php new file mode 100644 index 000000000000..1e8a93580e9f --- /dev/null +++ b/tests/unit/BigQuery/LoadJobConfigurationTest.php @@ -0,0 +1,121 @@ + self::PROJECT_ID, + 'datasetId' => self::DATASET_ID, + 'tableId' => self::TABLE_ID + ]; + private $expectedConfig; + + public function setUp() + { + $this->expectedConfig = [ + 'projectId' => self::PROJECT_ID, + 'jobReference' => [ + 'projectId' => self::PROJECT_ID, + 'jobId' => self::JOB_ID + ], + 'configuration' => [ + 'load' => [] + ] + ]; + $this->config = new LoadJobConfiguration( + self::PROJECT_ID, + ['jobReference' => ['jobId' => self::JOB_ID]] + ); + } + + public function testFluentSetters() + { + $destinationTable = $this->prophesize(Table::class); + $destinationTable->identity() + ->willReturn($this->tableIdentity); + $data = '1234'; + $load = [ + 'allowJaggedRows' => true, + 'allowQuotedNewlines' => true, + 'autodetect' => true, + 'createDisposition' => 'CREATE_NEVER', + 'destinationEncryptionConfiguration' => [ + 'kmsKeyName' => 'my_key' + ], + 'destinationTable' => $this->tableIdentity, + 'encoding' => 'UTF-8', + 'fieldDelimiter' => '\t', + 'ignoreUnknownValues' => true, + 'maxBadRecords' => 10, + 'nullMarker' => '\N', + 'projectionFields' => ['field_name'], + 'quote' => '"', + 'schema' => ['fields' => [['name' => 'col1', 'type' => 'STRING']]], + 'schemaUpdateOptions' => ['ALLOW_FIELD_ADDITION'], + 'skipLeadingRows' => 10, + 'sourceFormat' => 'CSV', + 'sourceUris' => ['gs://my_bucket/source.csv'], + 'timePartitioning' => [ + 'type' => 'DAY' + ], + 'writeDisposition' => 'WRITE_TRUNCATE' + ]; + $this->expectedConfig['configuration']['load'] = $load + + $this->expectedConfig['configuration']['load']; + $this->config + ->allowJaggedRows($load['allowJaggedRows']) + ->allowQuotedNewlines($load['allowQuotedNewlines']) + ->autodetect($load['autodetect']) + ->createDisposition($load['createDisposition']) + ->destinationEncryptionConfiguration($load['destinationEncryptionConfiguration']) + ->data($data) + ->destinationTable($destinationTable->reveal()) + ->encoding($load['encoding']) + ->fieldDelimiter($load['fieldDelimiter']) + ->ignoreUnknownValues($load['ignoreUnknownValues']) + ->maxBadRecords($load['maxBadRecords']) + ->nullMarker($load['nullMarker']) + ->projectionFields($load['projectionFields']) + ->quote($load['quote']) + ->schema($load['schema']) + ->schemaUpdateOptions($load['schemaUpdateOptions']) + ->skipLeadingRows($load['skipLeadingRows']) + ->sourceFormat($load['sourceFormat']) + ->sourceUris($load['sourceUris']) + ->timePartitioning($load['timePartitioning']) + ->writeDisposition($load['writeDisposition']); + + $this->assertEquals( + $this->expectedConfig + ['data' => $data], + $this->config->toArray() + ); + } +} diff --git a/tests/unit/BigQuery/QueryJobConfigurationTest.php b/tests/unit/BigQuery/QueryJobConfigurationTest.php new file mode 100644 index 000000000000..54ab16869c56 --- /dev/null +++ b/tests/unit/BigQuery/QueryJobConfigurationTest.php @@ -0,0 +1,189 @@ + self::PROJECT_ID, + 'datasetId' => self::DATASET_ID + ]; + private $tableIdentity = [ + 'projectId' => self::PROJECT_ID, + 'datasetId' => self::DATASET_ID, + 'tableId' => self::TABLE_ID + ]; + private $expectedConfig; + + public function setUp() + { + $this->expectedConfig = [ + 'projectId' => self::PROJECT_ID, + 'jobReference' => [ + 'projectId' => self::PROJECT_ID, + 'jobId' => self::JOB_ID + ], + 'configuration' => [ + 'query' => [ + 'useLegacySql' => false + ] + ] + ]; + $this->config = new QueryJobConfiguration( + new ValueMapper(false), + self::PROJECT_ID, + ['jobReference' => ['jobId' => self::JOB_ID]] + ); + } + + public function testFluentSetters() + { + $defaultDataset = $this->prophesize(Dataset::class); + $defaultDataset->identity() + ->willReturn($this->datasetIdentity); + $destinationTable = $this->prophesize(Table::class); + $destinationTable->identity() + ->willReturn($this->tableIdentity); + $query = [ + 'allowLargeResults' => true, + 'createDisposition' => 'CREATE_NEVER', + 'defaultDataset' => $this->datasetIdentity, + 'destinationEncryptionConfiguration' => [ + 'kmsKeyName' => 'my_key' + ], + 'destinationTable' => $this->tableIdentity, + 'flattenResults' => true, + 'maximumBillingTier' => 1, + 'maximumBytesBilled' => 100, + 'priority' => 'BATCH', + 'query' => 'SELECT * FROM test', + 'schemaUpdateOptions' => ['ALLOW_FIELD_ADDITION'], + 'tableDefinitions' => [ + 'autodetect' => true, + 'sourceUris' => [ + 'gs://my_bucket/table.json' + ] + ], + 'timePartitioning' => [ + 'type' => 'DAY' + ], + 'useLegacySql' => true, + 'useQueryCache' => true, + 'userDefinedFunctionResources' => [ + ['resourceUri' => 'gs://my_bucket/code_path'] + ], + 'writeDisposition' => 'WRITE_TRUNCATE' + ]; + $this->expectedConfig['configuration']['query'] = $query + + $this->expectedConfig['configuration']['query']; + $this->config + ->allowLargeResults($query['allowLargeResults']) + ->createDisposition($query['createDisposition']) + ->defaultDataset($defaultDataset->reveal()) + ->destinationEncryptionConfiguration($query['destinationEncryptionConfiguration']) + ->destinationTable($destinationTable->reveal()) + ->flattenResults($query['flattenResults']) + ->maximumBillingTier($query['maximumBillingTier']) + ->maximumBytesBilled($query['maximumBytesBilled']) + ->priority($query['priority']) + ->query($query['query']) + ->schemaUpdateOptions($query['schemaUpdateOptions']) + ->tableDefinitions($query['tableDefinitions']) + ->timePartitioning($query['timePartitioning']) + ->useLegacySql($query['useLegacySql']) + ->useQueryCache($query['useQueryCache']) + ->userDefinedFunctionResources($query['userDefinedFunctionResources']) + ->writeDisposition($query['writeDisposition']); + + $this->assertEquals($this->expectedConfig, $this->config->toArray()); + } + + /** + * @dataProvider parameterDataProvider + */ + public function testParameters($args, $expectedQuery) + { + $this->expectedConfig['configuration']['query'] = $expectedQuery + + $this->expectedConfig['configuration']['query']; + $this->config + ->parameters($args); + + $this->assertEquals($this->expectedConfig, $this->config->toArray()); + } + + public function parameterDataProvider() + { + return [ + [ + ['test' => 'parameter'], + [ + 'parameterMode' => 'named', + 'queryParameters' => [ + [ + 'name' => 'test', + 'parameterType' => [ + 'type' => 'STRING' + ], + 'parameterValue' => [ + 'value' => 'parameter' + ] + ] + ] + ] + ], + [ + [1, 2], + [ + 'parameterMode' => 'positional', + 'queryParameters' => [ + [ + 'parameterType' => [ + 'type' => 'INT64' + ], + 'parameterValue' => [ + 'value' => 1 + ] + ], + [ + 'parameterType' => [ + 'type' => 'INT64' + ], + 'parameterValue' => [ + 'value' => 2 + ] + ] + ] + ] + ] + ]; + } +} diff --git a/tests/unit/BigQuery/QueryResultsTest.php b/tests/unit/BigQuery/QueryResultsTest.php index 8f655d3a71aa..1900776fa78f 100644 --- a/tests/unit/BigQuery/QueryResultsTest.php +++ b/tests/unit/BigQuery/QueryResultsTest.php @@ -18,6 +18,7 @@ namespace Google\Cloud\Tests\Unit\BigQuery; use Google\Cloud\BigQuery\Connection\ConnectionInterface; +use Google\Cloud\BigQuery\Job; use Google\Cloud\BigQuery\QueryResults; use Google\Cloud\BigQuery\ValueMapper; use Prophecy\Argument; @@ -57,23 +58,11 @@ public function getQueryResults($connection, array $data = []) $this->jobId, $this->projectId, $data, - [], - new ValueMapper(false) + new ValueMapper(false), + $this->prophesize(Job::class)->reveal() ); } - /** - * @expectedException \Google\Cloud\Core\Exception\GoogleException - */ - public function testGetsRowsThrowsExceptionWhenQueryNotComplete() - { - $this->queryData['jobComplete'] = false; - unset($this->queryData['rows']); - $this->connection->getQueryResults()->shouldNotBeCalled(); - $queryResults = $this->getQueryResults($this->connection, $this->queryData); - $queryResults->rows()->next(); - } - public function testGetsRowsWithNoResults() { $this->connection->getQueryResults()->shouldNotBeCalled(); @@ -107,6 +96,27 @@ public function testGetsRowsWithToken() $this->assertEquals('Alton', $rows[1]['first_name']); } + public function testWaitsUntilComplete() + { + $this->queryData['jobComplete'] = false; + $this->connection->getQueryResults(Argument::any()) + ->willReturn([ + 'jobComplete' => true + ])->shouldBeCalledTimes(1); + $queryResults = $this->getQueryResults($this->connection, $this->queryData); + $queryResults->waitUntilComplete(); + } + + public function testGetIterator() + { + $this->connection->getQueryResults()->shouldNotBeCalled(); + unset($this->queryData['rows']); + $queryResults = $this->getQueryResults($this->connection, $this->queryData); + $rows = iterator_to_array($queryResults); + + $this->assertEmpty($rows); + } + public function testIsCompleteTrue() { $queryResults = $this->getQueryResults($this->connection, $this->queryData); @@ -122,6 +132,13 @@ public function testIsCompleteFalse() $this->assertFalse($queryResults->isComplete()); } + public function testGetsJob() + { + $queryResults = $this->getQueryResults($this->connection, $this->queryData); + + $this->assertInstanceOf(Job::class, $queryResults->job()); + } + public function testGetsInfo() { $queryResults = $this->getQueryResults($this->connection, $this->queryData); diff --git a/tests/unit/BigQuery/TableTest.php b/tests/unit/BigQuery/TableTest.php index 92c66d3bce82..2ac3caf640d7 100644 --- a/tests/unit/BigQuery/TableTest.php +++ b/tests/unit/BigQuery/TableTest.php @@ -18,10 +18,15 @@ namespace Google\Cloud\Tests\Unit\BigQuery; use Google\Cloud\BigQuery\Connection\ConnectionInterface; +use Google\Cloud\BigQuery\CopyJobConfiguration; +use Google\Cloud\BigQuery\ExtractJobConfiguration; use Google\Cloud\BigQuery\InsertResponse; use Google\Cloud\BigQuery\Job; +use Google\Cloud\BigQuery\JobConfigurationInterface; +use Google\Cloud\BigQuery\LoadJobConfiguration; use Google\Cloud\BigQuery\Table; use Google\Cloud\BigQuery\ValueMapper; +use Google\Cloud\Core\Exception\ConflictException; use Google\Cloud\Core\Exception\NotFoundException; use Google\Cloud\Core\Upload\AbstractUploader; use Google\Cloud\Storage\Connection\ConnectionInterface as StorageConnectionInterface; @@ -33,14 +38,16 @@ */ class TableTest extends \PHPUnit_Framework_TestCase { + const JOB_ID = 'myJobId'; + const PROJECT_ID = 'myProjectId'; + const BUCKET_NAME = 'myBucket'; + const FILE_NAME = 'myfile.csv'; + const TABLE_ID = 'myTableId'; + const DATASET_ID = 'myDatasetId'; + public $connection; public $storageConnection; public $mapper; - public $fileName = 'myfile.csv'; - public $bucketName = 'myBucket'; - public $projectId = 'myProjectId'; - public $tableId = 'myTableId'; - public $datasetId = 'myDatasetId'; public $rowData = [ 'rows' => [ ['f' => [['v' => 'Alton']]] @@ -58,7 +65,10 @@ class TableTest extends \PHPUnit_Framework_TestCase ]; public $insertJobResponse = [ 'jobReference' => [ - 'jobId' => 'myJobId' + 'jobId' => self::JOB_ID + ], + 'status' => [ + 'state' => 'RUNNING' ] ]; @@ -73,18 +83,18 @@ public function getObject() { return new StorageObject( $this->storageConnection->reveal(), - $this->fileName, - $this->bucketName + self::FILE_NAME, + self::BUCKET_NAME ); } public function getTable($connection, array $data = [], $tableId = null) { - return new Table( + return new TableStub( $connection->reveal(), - $tableId ?: $this->tableId, - $this->datasetId, - $this->projectId, + $tableId ?: self::TABLE_ID, + self::DATASET_ID, + self::PROJECT_ID, $this->mapper, $data ); @@ -94,7 +104,7 @@ public function testDoesExistTrue() { $this->connection->getTable(Argument::any()) ->willReturn([ - 'tableReference' => ['tableId' => $this->tableId] + 'tableReference' => ['tableId' => self::TABLE_ID] ]) ->shouldBeCalledTimes(1); $table = $this->getTable($this->connection); @@ -121,6 +131,22 @@ public function testDelete() } public function testUpdatesData() + { + $updateData = ['friendlyName' => 'wow a name', 'etag' => 'foo']; + $this->connection->patchTable(Argument::that(function ($args) { + if ($args['restOptions']['headers']['If-Match'] !== 'foo') return false; + + return true; + })) + ->willReturn($updateData) + ->shouldBeCalledTimes(1); + $table = $this->getTable($this->connection, ['friendlyName' => 'another name']); + $table->update($updateData); + + $this->assertEquals($updateData['friendlyName'], $table->info()['friendlyName']); + } + + public function testUpdatesDataWithEtag() { $updateData = ['friendlyName' => 'wow a name']; $this->connection->patchTable(Argument::any()) @@ -186,65 +212,145 @@ public function testGetsRowsWithToken() $this->assertEquals($name, $rows[1]['first_name']); } - public function testRunsCopyJob() + /** + * @dataProvider jobConfigDataProvider + */ + public function testRunJob($expectedData, $expectedMethod, $returnedData) + { + $jobConfig = $this->prophesize(JobConfigurationInterface::class); + $jobConfig->toArray() + ->willReturn($expectedData); + $this->connection->$expectedMethod($expectedData) + ->willReturn($returnedData) + ->shouldBeCalledTimes(1); + $this->connection->getJob(Argument::any()) + ->willReturn([ + 'status' => [ + 'state' => 'DONE' + ] + ]) + ->shouldBeCalledTimes(1); + $table = $this->getTable($this->connection); + $job = $table->runJob($jobConfig->reveal()); + + $this->assertInstanceOf(Job::class, $job); + $this->assertTrue($job->isComplete()); + } + + /** + * @dataProvider jobConfigDataProvider + */ + public function testStartJob($expectedData, $expectedMethod, $returnedData) + { + $jobConfig = $this->prophesize(JobConfigurationInterface::class); + $jobConfig->toArray() + ->willReturn($expectedData); + $this->connection->$expectedMethod($expectedData) + ->willReturn($returnedData) + ->shouldBeCalledTimes(1); + $this->connection->getJob(Argument::any()) + ->shouldNotBeCalled(); + $table = $this->getTable($this->connection); + $job = $table->startJob($jobConfig->reveal()); + + $this->assertInstanceOf(Job::class, $job); + $this->assertFalse($job->isComplete()); + $this->assertEquals($this->insertJobResponse, $job->info()); + } + + public function jobConfigDataProvider() + { + $expected = [ + 'projectId' => self::PROJECT_ID, + 'jobReference' => [ + 'projectId' => self::PROJECT_ID, + 'jobId' => self::JOB_ID + ] + ]; + $uploader = $this->prophesize(AbstractUploader::class); + $uploader->upload() + ->willReturn($this->insertJobResponse) + ->shouldBeCalledTimes(1); + + return [ + [ + $expected, + 'insertJob', + $this->insertJobResponse + ], + [ + $expected + ['data' => 'abc'], + 'insertJobUpload', + $uploader->reveal() + ] + ]; + } + + public function testGetsCopyJobConfiguration() { $destinationTableId = 'destinationTable'; $destinationTable = $this->getTable($this->connection, [], $destinationTableId); - $expectedArguments = [ - 'projectId' => $this->projectId, + $expected = [ + 'projectId' => self::PROJECT_ID, 'configuration' => [ 'copy' => [ 'destinationTable' => [ - 'datasetId' => $this->datasetId, + 'datasetId' => self::DATASET_ID, 'tableId' => $destinationTableId, - 'projectId' => $this->projectId + 'projectId' => self::PROJECT_ID ], 'sourceTable' => [ - 'datasetId' => $this->datasetId, - 'tableId' => $this->tableId, - 'projectId' => $this->projectId + 'datasetId' => self::DATASET_ID, + 'tableId' => self::TABLE_ID, + 'projectId' => self::PROJECT_ID ] ] + ], + 'jobReference' => [ + 'projectId' => self::PROJECT_ID, + 'jobId' => self::JOB_ID ] ]; - $this->connection->insertJob(Argument::exact($expectedArguments)) - ->willReturn($this->insertJobResponse) - ->shouldBeCalledTimes(1); $table = $this->getTable($this->connection); - $job = $table->copy($destinationTable); + $config = $table->copy($destinationTable, [ + 'jobReference' => ['jobId' => self::JOB_ID] + ]); - $this->assertInstanceOf(Job::class, $job); - $this->assertEquals($this->insertJobResponse, $job->info()); + $this->assertInstanceOf(CopyJobConfiguration::class, $config); + $this->assertEquals($expected, $config->toArray()); } /** * @dataProvider destinationProvider */ - public function testRunsExportJob($destinationObject) + public function testGetsExtractJobConfiguration($destinationObject) { - $expectedArguments = [ - 'projectId' => $this->projectId, + $expected = [ + 'projectId' => self::PROJECT_ID, 'configuration' => [ 'extract' => [ 'destinationUris' => [ - 'gs://' . $this->bucketName . '/' . $this->fileName + 'gs://' . self::BUCKET_NAME . '/' . self::FILE_NAME ], 'sourceTable' => [ - 'datasetId' => $this->datasetId, - 'tableId' => $this->tableId, - 'projectId' => $this->projectId + 'datasetId' => self::DATASET_ID, + 'tableId' => self::TABLE_ID, + 'projectId' => self::PROJECT_ID ] ] + ], + 'jobReference' => [ + 'projectId' => self::PROJECT_ID, + 'jobId' => self::JOB_ID ] ]; - $this->connection->insertJob(Argument::exact($expectedArguments)) - ->willReturn($this->insertJobResponse) - ->shouldBeCalledTimes(1); $table = $this->getTable($this->connection); - $job = $table->export($destinationObject); + $config = $table->extract($destinationObject, [ + 'jobReference' => ['jobId' => self::JOB_ID] + ]); - $this->assertInstanceOf(Job::class, $job); - $this->assertEquals($this->insertJobResponse, $job->info()); + $this->assertInstanceOf(ExtractJobConfiguration::class, $config); + $this->assertEquals($expected, $config->toArray()); } public function destinationProvider() @@ -255,68 +361,70 @@ public function destinationProvider() [$this->getObject()], [sprintf( 'gs://%s/%s', - $this->bucketName, - $this->fileName + self::BUCKET_NAME, + self::FILE_NAME )] ]; } - public function testRunsLoadJob() + public function testGetsLoadJobConfiguration() { $data = 'abc'; - $uploader = $this->prophesize(AbstractUploader::class); - $uploader->upload() - ->willReturn($this->insertJobResponse) - ->shouldBeCalledTimes(1); - $expectedArguments = [ + $expected = [ 'data' => $data, - 'projectId' => $this->projectId, + 'projectId' => self::PROJECT_ID, 'configuration' => [ 'load' => [ 'destinationTable' => [ - 'datasetId' => $this->datasetId, - 'tableId' => $this->tableId, - 'projectId' => $this->projectId + 'datasetId' => self::DATASET_ID, + 'tableId' => self::TABLE_ID, + 'projectId' => self::PROJECT_ID ] ] + ], + 'jobReference' => [ + 'projectId' => self::PROJECT_ID, + 'jobId' => self::JOB_ID ] ]; - $this->connection->insertJobUpload(Argument::exact($expectedArguments)) - ->willReturn($uploader) - ->shouldBeCalledTimes(1); $table = $this->getTable($this->connection); - $job = $table->load($data); + $config = $table->load($data, [ + 'jobReference' => ['jobId' => self::JOB_ID] + ]); - $this->assertInstanceOf(Job::class, $job); - $this->assertEquals($this->insertJobResponse, $job->info()); + $this->assertInstanceOf(LoadJobConfiguration::class, $config); + $this->assertEquals($expected, $config->toArray()); } - public function testRunsLoadJobFromStorage() + public function testGetsLoadJobConfigurationFromStorage() { $sourceObject = $this->getObject(); - $expectedArguments = [ - 'projectId' => $this->projectId, + $expected = [ + 'projectId' => self::PROJECT_ID, 'configuration' => [ 'load' => [ 'sourceUris' => [ - 'gs://' . $this->bucketName . '/' . $this->fileName + 'gs://' . self::BUCKET_NAME . '/' . self::FILE_NAME ], 'destinationTable' => [ - 'datasetId' => $this->datasetId, - 'tableId' => $this->tableId, - 'projectId' => $this->projectId + 'datasetId' => self::DATASET_ID, + 'tableId' => self::TABLE_ID, + 'projectId' => self::PROJECT_ID ] ] + ], + 'jobReference' => [ + 'projectId' => self::PROJECT_ID, + 'jobId' => self::JOB_ID ] ]; - $this->connection->insertJob(Argument::exact($expectedArguments)) - ->willReturn($this->insertJobResponse) - ->shouldBeCalledTimes(1); $table = $this->getTable($this->connection); - $job = $table->loadFromStorage($sourceObject); + $config = $table->loadFromStorage($sourceObject, [ + 'jobReference' => ['jobId' => self::JOB_ID] + ]); - $this->assertInstanceOf(Job::class, $job); - $this->assertEquals($this->insertJobResponse, $job->info()); + $this->assertInstanceOf(LoadJobConfiguration::class, $config); + $this->assertEquals($expected, $config->toArray()); } public function testInsertsRow() @@ -324,9 +432,9 @@ public function testInsertsRow() $insertId = '1'; $rowData = ['key' => 'value']; $expectedArguments = [ - 'tableId' => $this->tableId, - 'projectId' => $this->projectId, - 'datasetId' => $this->datasetId, + 'tableId' => self::TABLE_ID, + 'projectId' => self::PROJECT_ID, + 'datasetId' => self::DATASET_ID, 'rows' => [ [ 'json' => $rowData, @@ -358,9 +466,9 @@ public function testInsertsRows() ] ]; $expectedArguments = [ - 'tableId' => $this->tableId, - 'projectId' => $this->projectId, - 'datasetId' => $this->datasetId, + 'tableId' => self::TABLE_ID, + 'projectId' => self::PROJECT_ID, + 'datasetId' => self::DATASET_ID, 'rows' => [ [ 'json' => $data, @@ -372,25 +480,172 @@ public function testInsertsRows() ->willReturn([]) ->shouldBeCalledTimes(1); $table = $this->getTable($this->connection); - $insertResponse = $table->insertRows($rowData); $this->assertInstanceOf(InsertResponse::class, $insertResponse); $this->assertTrue($insertResponse->isSuccessful()); } + public function testInsertsRowsWithAutoCreate() + { + $insertId = '1'; + $data = ['key' => 'value']; + $rowData = [ + [ + 'insertId' => $insertId, + 'data' => $data + ] + ]; + $schema = [ + 'fields' => [ + [ + 'name' => 'key', + 'type' => 'STRING' + ] + ] + ]; + $expectedInsertTableDataArguments = [ + 'tableId' => self::TABLE_ID, + 'projectId' => self::PROJECT_ID, + 'datasetId' => self::DATASET_ID, + 'rows' => [ + [ + 'json' => $data, + 'insertId' => $insertId + ] + ] + ]; + $expectedInsertTableArguments = [ + 'schema' => $schema, + 'retries' => 0, + 'projectId' => self::PROJECT_ID, + 'datasetId' => self::DATASET_ID, + 'tableReference' => [ + 'projectId' => self::PROJECT_ID, + 'datasetId' => self::DATASET_ID, + 'tableId' => self::TABLE_ID + ] + ]; + $callCount = 0; + $this->connection->insertAllTableData($expectedInsertTableDataArguments) + ->will(function () use (&$callCount) { + if ($callCount === 0) { + $callCount++; + throw new NotFoundException(null); + } + + return []; + }) + ->shouldBeCalledTimes(2); + $this->connection->insertTable($expectedInsertTableArguments) + ->willReturn([]); + $table = $this->getTable($this->connection); + $insertResponse = $table->insertRows($rowData, [ + 'autoCreate' => true, + 'tableMetadata' => [ + 'schema' => $schema + ] + ]); + + $this->assertInstanceOf(InsertResponse::class, $insertResponse); + $this->assertTrue($insertResponse->isSuccessful()); + } + /** * @expectedException \InvalidArgumentException + * @expectedMessage A schema is required when creating a table. */ - public function testInsertRowsThrowsException() + public function testInsertRowsThrowsExceptionWithoutSchema() + { + $options = [ + 'autoCreate' => true + ]; + $this->connection->insertAllTableData(Argument::any()) + ->willThrow(new NotFoundException(null)); + $table = $this->getTable($this->connection); + $table->insertRows([ + [ + 'data' => [ + 'city' => 'state' + ] + ] + ], $options); + } + + /** + * @expectedException \Exception + */ + public function testInsertRowsThrowsExceptionWithUnretryableTableFailure() + { + $options = [ + 'autoCreate' => true, + 'tableMetadata' => [ + 'schema' => [] + ] + ]; + $this->connection->insertAllTableData(Argument::any()) + ->willThrow(new NotFoundException(null)); + $this->connection->insertTable(Argument::any()) + ->willThrow(new \Exception()); + $table = $this->getTable($this->connection); + $table->insertRows([ + [ + 'data' => [ + 'city' => 'state' + ] + ] + ], $options); + } + + /** + * @expectedException Google\Cloud\Core\Exception\NotFoundException + */ + public function testInsertRowsThrowsExceptionWhenMaxRetryLimitHit() + { + $options = [ + 'autoCreate' => true, + 'maxRetries' => 0, + 'tableMetadata' => [ + 'schema' => [] + ] + ]; + $this->connection->insertAllTableData(Argument::any()) + ->willThrow(new NotFoundException(null)); + $this->connection->insertTable(Argument::any()) + ->willThrow(new ConflictException(null)); + $table = $this->getTable($this->connection); + $table->insertRows([ + [ + 'data' => [ + 'city' => 'state' + ] + ] + ], $options); + } + + /** + * @expectedException \InvalidArgumentException + * @expectedMessage A row must have a data key. + */ + public function testInsertRowsThrowsExceptionWithoutDataKey() { $table = $this->getTable($this->connection); $table->insertRows([[], []]); } + /** + * @expectedException \InvalidArgumentException + * @expectedMessage Must provide at least a single row. + */ + public function testInsertRowsThrowsExceptionWithZeroRows() + { + $table = $this->getTable($this->connection); + $table->insertRows([]); + } + public function testGetsInfo() { - $tableInfo = ['tableReference' => ['tableId' => $this->tableId]]; + $tableInfo = ['tableReference' => ['tableId' => self::TABLE_ID]]; $this->connection->getTable(Argument::any())->shouldNotBeCalled(); $table = $this->getTable($this->connection, $tableInfo); @@ -399,7 +654,7 @@ public function testGetsInfo() public function testGetsInfoWithReload() { - $tableInfo = ['tableReference' => ['tableId' => $this->tableId]]; + $tableInfo = ['tableReference' => ['tableId' => self::TABLE_ID]]; $this->connection->getTable(Argument::any()) ->willReturn($tableInfo) ->shouldBeCalledTimes(1); @@ -412,14 +667,22 @@ public function testGetsId() { $table = $this->getTable($this->connection); - $this->assertEquals($this->tableId, $table->id()); + $this->assertEquals(self::TABLE_ID, $table->id()); } public function testGetsIdentity() { $table = $this->getTable($this->connection); - $this->assertEquals($this->tableId, $table->identity()['tableId']); - $this->assertEquals($this->projectId, $table->identity()['projectId']); + $this->assertEquals(self::TABLE_ID, $table->identity()['tableId']); + $this->assertEquals(self::PROJECT_ID, $table->identity()['projectId']); + } +} + +class TableStub extends Table +{ + protected function usleep($ms) + { + return; } } diff --git a/tests/unit/Core/ConcurrencyControlTraitTest.php b/tests/unit/Core/ConcurrencyControlTraitTest.php new file mode 100644 index 000000000000..3e1bc0e1fe74 --- /dev/null +++ b/tests/unit/Core/ConcurrencyControlTraitTest.php @@ -0,0 +1,53 @@ +trait = \Google\Cloud\Dev\impl(ConcurrencyControlTrait::class); + } + + public function testApplyEtagHeader() + { + $input = ['etag' => self::ETAG]; + + $res = $this->trait->call('applyEtagHeader', [$input]); + + $this->assertEquals(self::ETAG, $res['restOptions']['headers']['If-Match']); + } + + public function testApplyEtagHeaderCustomName() + { + $input = ['test' => self::ETAG]; + + $res = $this->trait->call('applyEtagHeader', [$input, 'test']); + + $this->assertEquals(self::ETAG, $res['restOptions']['headers']['If-Match']); + } +} diff --git a/tests/unit/Core/RetryDeciderTraitTest.php b/tests/unit/Core/RetryDeciderTraitTest.php index 4e3cc12842fe..608f0dab4bde 100644 --- a/tests/unit/Core/RetryDeciderTraitTest.php +++ b/tests/unit/Core/RetryDeciderTraitTest.php @@ -31,6 +31,18 @@ public function setUp() $this->implementation = new RetryDeciderTraitStub(); } + /** + * @dataProvider retryProvider + */ + public function testDisableRetry($exception) + { + $this->implementation->call('setHttpRetryCodes', [[]]); + $this->implementation->call('setHttpRetryMessages', [[]]); + $retryFunction = $this->implementation->call('getRetryFunction'); + + $this->assertFalse($retryFunction($exception)); + } + /** * @dataProvider retryProvider */