From bbe29c71b8d259b63a4ac7ff28c56e9de09e2d48 Mon Sep 17 00:00:00 2001 From: Ace Nassri Date: Mon, 15 Aug 2016 15:03:20 -0700 Subject: [PATCH 01/12] First draft of export samples --- bigquery/export_data.js | 145 +++++++++++++++ bigquery/package.json | 1 + bigquery/system-test/export_data.test.js | 57 ++++++ bigquery/test/export_data.test.js | 215 +++++++++++++++++++++++ 4 files changed, 418 insertions(+) create mode 100644 bigquery/export_data.js create mode 100644 bigquery/system-test/export_data.test.js create mode 100644 bigquery/test/export_data.test.js diff --git a/bigquery/export_data.js b/bigquery/export_data.js new file mode 100644 index 0000000000..52f79e4095 --- /dev/null +++ b/bigquery/export_data.js @@ -0,0 +1,145 @@ +// Copyright 2016, Google, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// [START complete] +/** + * Command-line application to export a table from BigQuery to Google Cloud Storage. + * + * This sample is used on this page: + * + * https://cloud.google.com/bigquery/exporting-data-from-bigquery + * For more information, see the README.md under /bigquery. + */ + +'use strict'; + +// [START auth] +// By default, gcloud will authenticate using the service account file specified +// by the GOOGLE_APPLICATION_CREDENTIALS environment variable and use the +// project specified by the GCLOUD_PROJECT environment variable. See +// https://googlecloudplatform.github.io/gcloud-node/#/docs/guides/authentication +var BigQuery = require('@google-cloud/bigquery'); +var Storage = require('@google-cloud/storage'); +var projectId = process.env.GCLOUD_PROJECT; + +// Instantiate the BigQuery and Storage clients +var bigquery = BigQuery(); +var storage = Storage({ + projectId: projectId +}); +// [END auth] + +// [START export_table_to_gcs] +/** + * Export a table from BigQuery to Google Cloud Storage. + * + * @param {object} options Configuration options. + * @param {string} options.bucket A Google Cloud Storage bucket to use for storage. + * @param {string} options.file The file to save results to within Google Cloud Storage. + * @param {string} options.dataset The ID of the dataset to use. + * @param {string} options.table The ID of the project to use. + * @param {string} options.format Format to export as - either 'CSV', 'JSON', or 'AVRO'. + * @param {boolean} [options.gzip] Optional. Whether or not data should be compressed using GZIP. + * @param {function} callback Callback function to receive query results. + */ +function exportTableToGCS (options, callback) { + var gcsFileObj = storage.bucket(options.bucket).file(options.file); + + // Export table + // See https://googlecloudplatform.github.io/gcloud-node/#/docs/google-cloud/latest/bigquery/table?method=export + var table = bigquery.dataset(options.dataset).table(options.table); + table.export( + gcsFileObj, + { format: options.format, gzip: options.gzip }, + function (err, job) { + if (err) { + return callback(err); + } + + console.log('ExportTableToGCS: submitted job %s!', job.id); + return callback(null, job); + } + ); +} +// [END export_table_to_gcs] + +// [START poll_export_job] +/** + * Check the status of a BigQuery table export job. + * + * @param {string} jobId The ID of the export job to poll. + * @param {function} callback Callback function to execute when the table is exported. + */ +function pollExportJob (jobId, callback) { + var job = bigquery.job(jobId); + job.getMetadata(function (err, metadata) { + if (err) { + return callback(err); + } + console.log('PollExportJob: job status: %s', metadata.status.state); + + // If job is done, return metadata; if not, return an error. + if (metadata.status.state === 'DONE') { + return callback(null, metadata); + } else { + return callback(new Error('Job %s is not done', jobId)); + } + }); +} +// [END poll_export_job] +// [END complete] + +// The command-line program +var cli = require('yargs'); + +var program = module.exports = { + exportTableToGCS: exportTableToGCS, + pollExportJob: pollExportJob, + main: function (args) { + // Run the command-line program + cli.help().strict().parse(args).argv; + } +}; + +cli + .command('export [--format] [--gzip]', 'Export a table from BigQuery to Google Cloud Storage.', {}, function (options) { + program.exportTableToGCS(options, console.log); + }) + .command('poll ', 'Check the status of a BigQuery table export job.', {}, function (options) { + program.pollExportJob(options.jobId, console.log); + }) + .example('node $0 export sample-bigquery-export data.json github_samples natality JSON', 'Export github_samples:natality to gcs://sample-bigquery-export/data.json as JSON') + .example('node $0 poll job_12345ABCDE', 'Check the status of BigQuery job 12345ABCDE') + .options({ + format: { + alias: 'f', + global: true, + requiresArg: true, + type: 'string', + choices: ['JSON', 'CSV', 'AVRO'] + }, + gzip: { + global: true, + type: 'boolean', + description: 'Whether to compress the exported table using gzip. Defaults to false.' + } + }) + .wrap(100) + .recommendCommands() + .epilogue('For more information, see https://cloud.google.com/bigquery/exporting-data-from-bigquery'); + +if (module === require.main) { + program.main(process.argv.slice(2), console.log); +} + +module.exports = program; diff --git a/bigquery/package.json b/bigquery/package.json index bdffb097da..42337d1645 100644 --- a/bigquery/package.json +++ b/bigquery/package.json @@ -11,6 +11,7 @@ "dependencies": { "@google-cloud/bigquery": "^0.1.1", "@google-cloud/resource": "^0.1.1", + "@google-cloud/storage": "^0.1.1", "async": "^2.0.1", "request": "^2.72.0" }, diff --git a/bigquery/system-test/export_data.test.js b/bigquery/system-test/export_data.test.js new file mode 100644 index 0000000000..58e4dc8226 --- /dev/null +++ b/bigquery/system-test/export_data.test.js @@ -0,0 +1,57 @@ +// Copyright 2016, Google, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +var example = require('../export_data'); +var options = { + bucket: 'sample-bigquery-export', + file: 'data.json', + dataset: 'github_samples', + table: 'natality' +}; +var jobId = null; + +describe('bigquery:export_data', function () { + describe('export_table_to_gcs', function () { + it('should export data to GCS', function (done) { + example.exportTableToGCS(options, function (err, job) { + assert.ifError(err); + assert.notEqual(job, null); + assert.notEqual(job.id, null); + assert(job.id.length > 5); + jobId = job.id; + setTimeout(done, 100); // Wait for export job to be submitted + }); + }); + }); + + describe('export_poll', function () { + it('should fetch job status', function (done) { + assert.notEqual(jobId, null); + var poller = function (tries) { + example.pollExportJob(jobId, function (err, metadata) { + if (!err || tries === 0) { + assert.ifError(err); + assert.equal(metadata.status.state, 'DONE'); + done(); + } else { + setTimeout(function () { poller(tries - 1); }, 1000); + } + }); + }; + + poller(60); + }); + }); +}); diff --git a/bigquery/test/export_data.test.js b/bigquery/test/export_data.test.js new file mode 100644 index 0000000000..7901e4ae02 --- /dev/null +++ b/bigquery/test/export_data.test.js @@ -0,0 +1,215 @@ +// Copyright 2016, Google, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +var proxyquire = require('proxyquire').noCallThru(); + +function getSample () { + var bucketMock = { + file: sinon.stub().returns(fileMock) + }; + + var storageMock = { + bucket: sinon.stub().returns(bucketMock) + }; + + var fileMock = {}; + + var metadataMock = { status: { state: 'DONE' } }; + + var jobId = 'abc'; + + var jobMock = { + id: jobId, + getMetadata: sinon.stub().callsArgWith(0, null, metadataMock) + }; + + var tableMock = { + export: sinon.stub().callsArgWith(2, null, jobMock) + }; + + var datasetMock = { + table: sinon.stub().returns(tableMock) + }; + + var bigqueryMock = { + job: sinon.stub().returns(jobMock), + dataset: sinon.stub().returns(datasetMock) + }; + + var BigQueryMock = sinon.stub().returns(bigqueryMock); + var StorageMock = sinon.stub().returns(storageMock); + + return { + program: proxyquire('../export_data', { + '@google-cloud/bigquery': BigQueryMock, + '@google-cloud/storage': StorageMock, + yargs: proxyquire('yargs', {}) + }), + mocks: { + BigQuery: BigQueryMock, + bigquery: bigqueryMock, + Storage: StorageMock, + storage: storageMock, + metadata: metadataMock, + job: jobMock, + table: tableMock, + bucket: bucketMock, + dataset: datasetMock + }, + jobId: jobId + }; +} + +describe('bigquery:export', function () { + describe('exportTable', function () { + it('should export to a table', function () { + var example = getSample(); + var options = { + bucket: 'bucket', + file: 'file', + dataset: 'dataset', + table: 'table', + format: 'JSON', + gzip: true + }; + + var callback = sinon.stub(); + example.program.exportTableToGCS(options, callback); + + assert(example.mocks.storage.bucket.calledWith(options.bucket), 'bucket found'); + assert(example.mocks.bucket.file.calledWith(options.file), 'file found'); + assert(example.mocks.bigquery.dataset.calledWith(options.dataset), 'dataset found'); + assert(example.mocks.dataset.table.calledWith(options.table), 'table found'); + assert(example.mocks.table.export.calledOnce, 'table.export called once'); + + assert(console.log.calledWith('ExportTableToGCS: submitted job %s!', example.mocks.job.id), + 'job submittal was reported' + ); + + assert.equal(callback.firstCall.args.length, 2, 'callback has 2 arguments'); + assert.ifError(callback.firstCall.args[0], 'callback did not receive error'); + assert.equal(callback.firstCall.args[1], example.mocks.job, 'callback received job'); + }); + + it('should handle error', function () { + var error = new Error('exportTableToGCSError'); + var example = getSample(); + var callback = sinon.stub(); + example.mocks.table.export = sinon.stub().callsArgWith(2, error); + example.program.exportTableToGCS({ format: 'JSON' }, callback); + + assert(callback.calledOnce, 'callback called once'); + assert.equal(callback.firstCall.args.length, 1, 'callback received 1 argument'); + assert.equal(callback.firstCall.args[0], error, 'callback received error'); + }); + }); + + describe('pollExportJob', function () { + it('should fetch and report the status of a job', function () { + var example = getSample(); + var callback = sinon.stub(); + example.mocks.bigquery.job = sinon.stub().returns(example.mocks.job); + example.program.pollExportJob(example.jobId, callback); + + assert(example.mocks.bigquery.job.calledOnce); + assert(example.mocks.job.getMetadata.calledOnce); + assert(callback.calledOnce); + assert.equal(callback.firstCall.args.length, 2); + assert.equal(callback.firstCall.args[0], null); + assert.deepEqual(callback.firstCall.args[1], example.mocks.metadata); + + assert( + console.log.calledWith('PollExportJob: job status: %s', example.mocks.metadata.status.state), + 'job status was reported' + ); + }); + + it('should error if a job is not finished', function () { + var example = getSample(); + var callback = sinon.stub(); + + var pendingState = { status: { state: 'PENDING' } }; + example.mocks.job.getMetadata = sinon.stub().callsArgWith(0, null, pendingState); + example.program.pollExportJob(example.jobId, callback); + + assert(example.mocks.bigquery.job.calledOnce, 'bigquery.job called once'); + assert(example.mocks.job.getMetadata.calledOnce, 'getMetadata called once'); + assert(callback.calledOnce, 'callback called once'); + assert.equal(callback.firstCall.args.length, 1, 'callback received 1 argument'); + assert.deepEqual( + callback.firstCall.args[0], + new Error('Job %s is not done'), + 'callback received error' + ); + + assert(console.log.calledWith('PollExportJob: job status: %s', pendingState.status.state), + 'job status was reported' + ); + + var doneState = { status: { state: 'DONE' } }; + example.mocks.job.getMetadata = sinon.stub().callsArgWith(0, null, doneState); + example.program.pollExportJob(example.jobId, callback); + + assert(example.mocks.bigquery.job.calledTwice, 'bigquery.job called a second time'); + assert(example.mocks.job.getMetadata.calledOnce, 'new getMetadata called once'); + assert(callback.calledTwice, 'callback called a second time'); + + assert.equal(callback.secondCall.args.length, 2, 'callback received 2 arguments'); + assert.ifError(callback.secondCall.args[0], 'callback did not receive error'); + assert.deepEqual( + callback.secondCall.args[1], + example.mocks.metadata, + 'callback received metadata' + ); + + assert(console.log.calledWith('PollExportJob: job status: %s', doneState.status.state), + 'job status was reported' + ); + }); + + it('should handle error', function () { + var error = new Error('pollExportJobError'); + var example = getSample(); + var callback = sinon.stub(); + example.mocks.job.getMetadata = sinon.stub().callsArgWith(0, error); + example.program.pollExportJob(example.jobId, callback); + + assert(example.mocks.bigquery.job.calledOnce, 'bigquery.job called once'); + assert(example.mocks.job.getMetadata.calledOnce, 'getMetadata called once'); + assert(callback.calledOnce, 'callback called'); + assert.equal(callback.firstCall.args.length, 1, 'callback received 1 argument'); + assert.equal(callback.firstCall.args[0], error, 'callback received error'); + }); + }); + + describe('main', function () { + it('should call exportTableToGCS', function () { + var program = getSample().program; + program.exportTableToGCS = sinon.stub(); + + program.main(['export', 'bucket', 'file', 'dataset', 'table', 'JSON']); + assert(program.exportTableToGCS.calledOnce); + }); + + it('should call pollExportJob', function () { + var program = getSample().program; + program.pollExportJob = sinon.stub(); + var jobId = 'ABCDE'; + + program.main(['poll', jobId]); + assert(program.pollExportJob.calledWith(jobId)); + }); + }); +}); From 3d35fac0933e264974b5d77657ff99ae5fd0122c Mon Sep 17 00:00:00 2001 From: Ace Nassri Date: Mon, 22 Aug 2016 12:48:56 -0700 Subject: [PATCH 02/12] Add yargs --- bigquery/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bigquery/package.json b/bigquery/package.json index 42337d1645..07c3033af1 100644 --- a/bigquery/package.json +++ b/bigquery/package.json @@ -13,7 +13,8 @@ "@google-cloud/resource": "^0.1.1", "@google-cloud/storage": "^0.1.1", "async": "^2.0.1", - "request": "^2.72.0" + "request": "^2.72.0", + "yargs": "^5.0.0" }, "devDependencies": { "mocha": "^3.0.2" From bbd5ee854d4229e9a4c245624f85f2a89e5b2e62 Mon Sep 17 00:00:00 2001 From: Ace Nassri Date: Mon, 22 Aug 2016 12:57:16 -0700 Subject: [PATCH 03/12] Minor stylistic tweaks --- bigquery/system-test/export_data.test.js | 16 +++++++++------- bigquery/test/export_data.test.js | 22 ++++++++++------------ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/bigquery/system-test/export_data.test.js b/bigquery/system-test/export_data.test.js index 58e4dc8226..5b7fc0b498 100644 --- a/bigquery/system-test/export_data.test.js +++ b/bigquery/system-test/export_data.test.js @@ -27,9 +27,9 @@ describe('bigquery:export_data', function () { it('should export data to GCS', function (done) { example.exportTableToGCS(options, function (err, job) { assert.ifError(err); - assert.notEqual(job, null); - assert.notEqual(job.id, null); - assert(job.id.length > 5); + assert(job, 'job is not null'); + assert(job.id, 'job has an id'); + assert(job.id.length > 5, 'job id is 5 characters or more'); jobId = job.id; setTimeout(done, 100); // Wait for export job to be submitted }); @@ -38,15 +38,17 @@ describe('bigquery:export_data', function () { describe('export_poll', function () { it('should fetch job status', function (done) { - assert.notEqual(jobId, null); + assert(jobId); var poller = function (tries) { example.pollExportJob(jobId, function (err, metadata) { if (!err || tries === 0) { - assert.ifError(err); - assert.equal(metadata.status.state, 'DONE'); + assert.ifError(err, 'no error occurred'); + assert.equal(metadata.status.state, 'DONE', 'export job is finished'); done(); } else { - setTimeout(function () { poller(tries - 1); }, 1000); + setTimeout(function () { + poller(tries - 1); + }, 1000); } }); }; diff --git a/bigquery/test/export_data.test.js b/bigquery/test/export_data.test.js index 7901e4ae02..ed75eb1ff5 100644 --- a/bigquery/test/export_data.test.js +++ b/bigquery/test/export_data.test.js @@ -123,12 +123,14 @@ describe('bigquery:export', function () { example.mocks.bigquery.job = sinon.stub().returns(example.mocks.job); example.program.pollExportJob(example.jobId, callback); - assert(example.mocks.bigquery.job.calledOnce); - assert(example.mocks.job.getMetadata.calledOnce); - assert(callback.calledOnce); - assert.equal(callback.firstCall.args.length, 2); - assert.equal(callback.firstCall.args[0], null); - assert.deepEqual(callback.firstCall.args[1], example.mocks.metadata); + assert(example.mocks.bigquery.job.calledOnce, 'job called once'); + assert(example.mocks.job.getMetadata.calledOnce, 'getMetadata called once'); + assert(callback.calledOnce, 'callback called once'); + assert.equal(callback.firstCall.args.length, 2, 'callback received 2 arguments'); + assert.equal(callback.firstCall.args[0], null, 'callback did not receive error'); + assert.deepEqual(callback.firstCall.args[1], example.mocks.metadata, + 'callback received metadata' + ); assert( console.log.calledWith('PollExportJob: job status: %s', example.mocks.metadata.status.state), @@ -148,9 +150,7 @@ describe('bigquery:export', function () { assert(example.mocks.job.getMetadata.calledOnce, 'getMetadata called once'); assert(callback.calledOnce, 'callback called once'); assert.equal(callback.firstCall.args.length, 1, 'callback received 1 argument'); - assert.deepEqual( - callback.firstCall.args[0], - new Error('Job %s is not done'), + assert.deepEqual(callback.firstCall.args[0], new Error('Job %s is not done'), 'callback received error' ); @@ -168,9 +168,7 @@ describe('bigquery:export', function () { assert.equal(callback.secondCall.args.length, 2, 'callback received 2 arguments'); assert.ifError(callback.secondCall.args[0], 'callback did not receive error'); - assert.deepEqual( - callback.secondCall.args[1], - example.mocks.metadata, + assert.deepEqual(callback.secondCall.args[1], example.mocks.metadata, 'callback received metadata' ); From 29df1dbc564c1620ce5ebb20efadbdf2fb13c87e Mon Sep 17 00:00:00 2001 From: Ace Nassri Date: Mon, 22 Aug 2016 13:18:37 -0700 Subject: [PATCH 04/12] Move export_data to tables --- bigquery/system-test/{export_data.test.js => tables.test.js} | 4 ++-- bigquery/{export_data.js => tables.js} | 0 bigquery/test/{export_data.test.js => tables.test.js} | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename bigquery/system-test/{export_data.test.js => tables.test.js} (95%) rename bigquery/{export_data.js => tables.js} (100%) rename bigquery/test/{export_data.test.js => tables.test.js} (98%) diff --git a/bigquery/system-test/export_data.test.js b/bigquery/system-test/tables.test.js similarity index 95% rename from bigquery/system-test/export_data.test.js rename to bigquery/system-test/tables.test.js index 5b7fc0b498..16a1fbe995 100644 --- a/bigquery/system-test/export_data.test.js +++ b/bigquery/system-test/tables.test.js @@ -13,7 +13,7 @@ 'use strict'; -var example = require('../export_data'); +var example = require('../tables'); var options = { bucket: 'sample-bigquery-export', file: 'data.json', @@ -22,7 +22,7 @@ var options = { }; var jobId = null; -describe('bigquery:export_data', function () { +describe('bigquery:tables', function () { describe('export_table_to_gcs', function () { it('should export data to GCS', function (done) { example.exportTableToGCS(options, function (err, job) { diff --git a/bigquery/export_data.js b/bigquery/tables.js similarity index 100% rename from bigquery/export_data.js rename to bigquery/tables.js diff --git a/bigquery/test/export_data.test.js b/bigquery/test/tables.test.js similarity index 98% rename from bigquery/test/export_data.test.js rename to bigquery/test/tables.test.js index ed75eb1ff5..6979a6be4c 100644 --- a/bigquery/test/export_data.test.js +++ b/bigquery/test/tables.test.js @@ -52,7 +52,7 @@ function getSample () { var StorageMock = sinon.stub().returns(storageMock); return { - program: proxyquire('../export_data', { + program: proxyquire('../tables', { '@google-cloud/bigquery': BigQueryMock, '@google-cloud/storage': StorageMock, yargs: proxyquire('yargs', {}) @@ -72,7 +72,7 @@ function getSample () { }; } -describe('bigquery:export', function () { +describe('bigquery:tables', function () { describe('exportTable', function () { it('should export to a table', function () { var example = getSample(); From f2b5d96c6146bb9bca3c0795b34036bb00aad0a2 Mon Sep 17 00:00:00 2001 From: Ace Nassri Date: Mon, 22 Aug 2016 14:06:47 -0700 Subject: [PATCH 05/12] Address comments --- bigquery/tables.js | 27 ++++++++++++--------------- bigquery/test/tables.test.js | 23 ++++++----------------- 2 files changed, 18 insertions(+), 32 deletions(-) diff --git a/bigquery/tables.js b/bigquery/tables.js index 52f79e4095..8c9c09fc2c 100644 --- a/bigquery/tables.js +++ b/bigquery/tables.js @@ -30,13 +30,10 @@ // https://googlecloudplatform.github.io/gcloud-node/#/docs/guides/authentication var BigQuery = require('@google-cloud/bigquery'); var Storage = require('@google-cloud/storage'); -var projectId = process.env.GCLOUD_PROJECT; // Instantiate the BigQuery and Storage clients var bigquery = BigQuery(); -var storage = Storage({ - projectId: projectId -}); +var storage = Storage(); // [END auth] // [START export_table_to_gcs] @@ -60,7 +57,10 @@ function exportTableToGCS (options, callback) { var table = bigquery.dataset(options.dataset).table(options.table); table.export( gcsFileObj, - { format: options.format, gzip: options.gzip }, + { + format: options.format, + gzip: options.gzip + }, function (err, job) { if (err) { return callback(err); @@ -112,15 +112,10 @@ var program = module.exports = { }; cli - .command('export
[--format] [--gzip]', 'Export a table from BigQuery to Google Cloud Storage.', {}, function (options) { + .command('export
', 'Export a table from BigQuery to Google Cloud Storage.', {}, function (options) { program.exportTableToGCS(options, console.log); }) - .command('poll ', 'Check the status of a BigQuery table export job.', {}, function (options) { - program.pollExportJob(options.jobId, console.log); - }) - .example('node $0 export sample-bigquery-export data.json github_samples natality JSON', 'Export github_samples:natality to gcs://sample-bigquery-export/data.json as JSON') - .example('node $0 poll job_12345ABCDE', 'Check the status of BigQuery job 12345ABCDE') - .options({ + .command('poll ', 'Check the status of a BigQuery table export job.', { format: { alias: 'f', global: true, @@ -133,13 +128,15 @@ cli type: 'boolean', description: 'Whether to compress the exported table using gzip. Defaults to false.' } + }, function (options) { + program.pollExportJob(options.jobId, console.log); }) + .example('node $0 export sample-bigquery-export data.json github_samples natality JSON', 'Export github_samples:natality to gcs://sample-bigquery-export/data.json as JSON') + .example('node $0 poll job_12345ABCDE', 'Check the status of BigQuery job 12345ABCDE') .wrap(100) .recommendCommands() .epilogue('For more information, see https://cloud.google.com/bigquery/exporting-data-from-bigquery'); if (module === require.main) { - program.main(process.argv.slice(2), console.log); + program.main(process.argv.slice(2)); } - -module.exports = program; diff --git a/bigquery/test/tables.test.js b/bigquery/test/tables.test.js index 6979a6be4c..3dbd0ceebe 100644 --- a/bigquery/test/tables.test.js +++ b/bigquery/test/tables.test.js @@ -14,40 +14,30 @@ 'use strict'; var proxyquire = require('proxyquire').noCallThru(); +var jobId = 'abc'; function getSample () { var bucketMock = { file: sinon.stub().returns(fileMock) }; - var storageMock = { bucket: sinon.stub().returns(bucketMock) }; - var fileMock = {}; - var metadataMock = { status: { state: 'DONE' } }; - - var jobId = 'abc'; - var jobMock = { - id: jobId, getMetadata: sinon.stub().callsArgWith(0, null, metadataMock) }; - var tableMock = { export: sinon.stub().callsArgWith(2, null, jobMock) }; - var datasetMock = { table: sinon.stub().returns(tableMock) }; - var bigqueryMock = { job: sinon.stub().returns(jobMock), dataset: sinon.stub().returns(datasetMock) }; - var BigQueryMock = sinon.stub().returns(bigqueryMock); var StorageMock = sinon.stub().returns(storageMock); @@ -67,8 +57,7 @@ function getSample () { table: tableMock, bucket: bucketMock, dataset: datasetMock - }, - jobId: jobId + } }; } @@ -121,7 +110,7 @@ describe('bigquery:tables', function () { var example = getSample(); var callback = sinon.stub(); example.mocks.bigquery.job = sinon.stub().returns(example.mocks.job); - example.program.pollExportJob(example.jobId, callback); + example.program.pollExportJob(jobId, callback); assert(example.mocks.bigquery.job.calledOnce, 'job called once'); assert(example.mocks.job.getMetadata.calledOnce, 'getMetadata called once'); @@ -144,7 +133,7 @@ describe('bigquery:tables', function () { var pendingState = { status: { state: 'PENDING' } }; example.mocks.job.getMetadata = sinon.stub().callsArgWith(0, null, pendingState); - example.program.pollExportJob(example.jobId, callback); + example.program.pollExportJob(jobId, callback); assert(example.mocks.bigquery.job.calledOnce, 'bigquery.job called once'); assert(example.mocks.job.getMetadata.calledOnce, 'getMetadata called once'); @@ -160,7 +149,7 @@ describe('bigquery:tables', function () { var doneState = { status: { state: 'DONE' } }; example.mocks.job.getMetadata = sinon.stub().callsArgWith(0, null, doneState); - example.program.pollExportJob(example.jobId, callback); + example.program.pollExportJob(jobId, callback); assert(example.mocks.bigquery.job.calledTwice, 'bigquery.job called a second time'); assert(example.mocks.job.getMetadata.calledOnce, 'new getMetadata called once'); @@ -182,7 +171,7 @@ describe('bigquery:tables', function () { var example = getSample(); var callback = sinon.stub(); example.mocks.job.getMetadata = sinon.stub().callsArgWith(0, error); - example.program.pollExportJob(example.jobId, callback); + example.program.pollExportJob(jobId, callback); assert(example.mocks.bigquery.job.calledOnce, 'bigquery.job called once'); assert(example.mocks.job.getMetadata.calledOnce, 'getMetadata called once'); From 05a1de0c41009fd0de8a8d8fb2538d6f3f7cad43 Mon Sep 17 00:00:00 2001 From: Ace Nassri Date: Tue, 23 Aug 2016 18:52:10 -0700 Subject: [PATCH 06/12] Address verbal comments --- bigquery/system-test/tables.test.js | 80 ++++++++++++++--------- bigquery/tables.js | 45 +++---------- bigquery/test/tables.test.js | 98 +++++------------------------ 3 files changed, 74 insertions(+), 149 deletions(-) diff --git a/bigquery/system-test/tables.test.js b/bigquery/system-test/tables.test.js index 16a1fbe995..8748ccb110 100644 --- a/bigquery/system-test/tables.test.js +++ b/bigquery/system-test/tables.test.js @@ -13,47 +13,67 @@ 'use strict'; +var uuid = require('node-uuid'); +var generateUuid = function () { + return 'nodejs_docs_samples_' + uuid.v4().replace(/-/gi, '_'); +}; var example = require('../tables'); var options = { - bucket: 'sample-bigquery-export', + bucket: generateUuid(), file: 'data.json', - dataset: 'github_samples', - table: 'natality' + dataset: generateUuid(), + table: generateUuid() }; -var jobId = null; +var BigQuery = require('@google-cloud/bigquery'); +var bigquery = BigQuery(); +var Storage = require('@google-cloud/storage'); +var storage = Storage(); +var file = storage.bucket(options.bucket).file(options.file); describe('bigquery:tables', function () { - describe('export_table_to_gcs', function () { - it('should export data to GCS', function (done) { - example.exportTableToGCS(options, function (err, job) { - assert.ifError(err); - assert(job, 'job is not null'); - assert(job.id, 'job has an id'); - assert(job.id.length > 5, 'job id is 5 characters or more'); - jobId = job.id; - setTimeout(done, 100); // Wait for export job to be submitted + before(function (done) { + // Create bucket + storage.createBucket(options.bucket, function (err, bucket) { + assert.ifError(err, 'bucket creation succeeded'); + + // Create dataset + bigquery.createDataset(options.dataset, function (err, dataset) { + assert.ifError(err, 'dataset creation succeeded'); + + // Create table + dataset.createTable( + options.table, + { schema: 'name:string, age:integer' }, + function (err, table) { + assert.ifError(err, 'table creation succeeded'); + done(); + } + ); }); }); }); + after(function (done) { + // Delete table export + file.delete(function () { + // Delete testing dataset/table + bigquery.dataset(options.dataset).delete({ force: true }, done); + }); + }); - describe('export_poll', function () { - it('should fetch job status', function (done) { - assert(jobId); - var poller = function (tries) { - example.pollExportJob(jobId, function (err, metadata) { - if (!err || tries === 0) { - assert.ifError(err, 'no error occurred'); - assert.equal(metadata.status.state, 'DONE', 'export job is finished'); - done(); - } else { - setTimeout(function () { - poller(tries - 1); - }, 1000); - } - }); - }; + describe('export_table_to_gcs', function () { + it('should export data to GCS', function (done) { + example.exportTableToGCS(options, function (err, metadata) { + assert.ifError(err, 'no error occurred'); + assert(metadata, 'job metadata was received'); + assert(metadata.status, 'job metadata has status'); + assert.equal(metadata.status.state, 'DONE', 'job was finished'); - poller(60); + file.exists(function (err, exists) { + assert.ifError(err, 'file existence check succeeded'); + assert(exists, 'export destination exists'); + done(); + }); + }); }); }); }); diff --git a/bigquery/tables.js b/bigquery/tables.js index 8c9c09fc2c..b26df361f5 100644 --- a/bigquery/tables.js +++ b/bigquery/tables.js @@ -65,38 +65,18 @@ function exportTableToGCS (options, callback) { if (err) { return callback(err); } - console.log('ExportTableToGCS: submitted job %s!', job.id); - return callback(null, job); + + job.on('error', function (err) { + return callback(err); + }); + job.on('complete', function (job) { + return callback(null, job); + }); } ); } // [END export_table_to_gcs] - -// [START poll_export_job] -/** - * Check the status of a BigQuery table export job. - * - * @param {string} jobId The ID of the export job to poll. - * @param {function} callback Callback function to execute when the table is exported. - */ -function pollExportJob (jobId, callback) { - var job = bigquery.job(jobId); - job.getMetadata(function (err, metadata) { - if (err) { - return callback(err); - } - console.log('PollExportJob: job status: %s', metadata.status.state); - - // If job is done, return metadata; if not, return an error. - if (metadata.status.state === 'DONE') { - return callback(null, metadata); - } else { - return callback(new Error('Job %s is not done', jobId)); - } - }); -} -// [END poll_export_job] // [END complete] // The command-line program @@ -104,7 +84,6 @@ var cli = require('yargs'); var program = module.exports = { exportTableToGCS: exportTableToGCS, - pollExportJob: pollExportJob, main: function (args) { // Run the command-line program cli.help().strict().parse(args).argv; @@ -112,10 +91,7 @@ var program = module.exports = { }; cli - .command('export
', 'Export a table from BigQuery to Google Cloud Storage.', {}, function (options) { - program.exportTableToGCS(options, console.log); - }) - .command('poll ', 'Check the status of a BigQuery table export job.', { + .command('export
', 'Export a table from BigQuery to Google Cloud Storage.', { format: { alias: 'f', global: true, @@ -129,10 +105,9 @@ cli description: 'Whether to compress the exported table using gzip. Defaults to false.' } }, function (options) { - program.pollExportJob(options.jobId, console.log); + program.exportTableToGCS(options, console.log); }) - .example('node $0 export sample-bigquery-export data.json github_samples natality JSON', 'Export github_samples:natality to gcs://sample-bigquery-export/data.json as JSON') - .example('node $0 poll job_12345ABCDE', 'Check the status of BigQuery job 12345ABCDE') + .example('node $0 export sample-bigquery-export data.json github_samples natality JSON --gzip', 'Export github_samples:natality to gcs://sample-bigquery-export/data.json as gzipped JSON') .wrap(100) .recommendCommands() .epilogue('For more information, see https://cloud.google.com/bigquery/exporting-data-from-bigquery'); diff --git a/bigquery/test/tables.test.js b/bigquery/test/tables.test.js index 3dbd0ceebe..b10af95ee8 100644 --- a/bigquery/test/tables.test.js +++ b/bigquery/test/tables.test.js @@ -14,7 +14,6 @@ 'use strict'; var proxyquire = require('proxyquire').noCallThru(); -var jobId = 'abc'; function getSample () { var bucketMock = { @@ -26,8 +25,10 @@ function getSample () { var fileMock = {}; var metadataMock = { status: { state: 'DONE' } }; var jobMock = { - getMetadata: sinon.stub().callsArgWith(0, null, metadataMock) + getMetadata: sinon.stub().callsArgWith(0, null, metadataMock), + on: sinon.stub() }; + var tableMock = { export: sinon.stub().callsArgWith(2, null, jobMock) }; @@ -73,8 +74,10 @@ describe('bigquery:tables', function () { format: 'JSON', gzip: true }; - var callback = sinon.stub(); + example.mocks.job.on.withArgs('complete').callsArgWith(1, example.mocks.metadata); + example.mocks.job.on.withArgs('error').returns(0); + example.program.exportTableToGCS(options, callback); assert(example.mocks.storage.bucket.calledWith(options.bucket), 'bucket found'); @@ -82,17 +85,17 @@ describe('bigquery:tables', function () { assert(example.mocks.bigquery.dataset.calledWith(options.dataset), 'dataset found'); assert(example.mocks.dataset.table.calledWith(options.table), 'table found'); assert(example.mocks.table.export.calledOnce, 'table.export called once'); - assert(console.log.calledWith('ExportTableToGCS: submitted job %s!', example.mocks.job.id), 'job submittal was reported' ); - assert.equal(callback.firstCall.args.length, 2, 'callback has 2 arguments'); + assert(callback.calledOnce, 'callback called once'); + assert.equal(callback.firstCall.args.length, 2, 'callback received 2 arguments'); assert.ifError(callback.firstCall.args[0], 'callback did not receive error'); - assert.equal(callback.firstCall.args[1], example.mocks.job, 'callback received job'); + assert.equal(callback.firstCall.args[1], example.mocks.metadata, 'callback received metadata'); }); - it('should handle error', function () { + it('should handle export error', function () { var error = new Error('exportTableToGCSError'); var example = getSample(); var callback = sinon.stub(); @@ -103,80 +106,16 @@ describe('bigquery:tables', function () { assert.equal(callback.firstCall.args.length, 1, 'callback received 1 argument'); assert.equal(callback.firstCall.args[0], error, 'callback received error'); }); - }); - - describe('pollExportJob', function () { - it('should fetch and report the status of a job', function () { - var example = getSample(); - var callback = sinon.stub(); - example.mocks.bigquery.job = sinon.stub().returns(example.mocks.job); - example.program.pollExportJob(jobId, callback); - - assert(example.mocks.bigquery.job.calledOnce, 'job called once'); - assert(example.mocks.job.getMetadata.calledOnce, 'getMetadata called once'); - assert(callback.calledOnce, 'callback called once'); - assert.equal(callback.firstCall.args.length, 2, 'callback received 2 arguments'); - assert.equal(callback.firstCall.args[0], null, 'callback did not receive error'); - assert.deepEqual(callback.firstCall.args[1], example.mocks.metadata, - 'callback received metadata' - ); - - assert( - console.log.calledWith('PollExportJob: job status: %s', example.mocks.metadata.status.state), - 'job status was reported' - ); - }); - it('should error if a job is not finished', function () { + it('should handle job-processing error', function () { + var error = new Error('exportTableToGCSError'); var example = getSample(); var callback = sinon.stub(); + example.mocks.job.on.withArgs('error').callsArgWith(1, error); + example.program.exportTableToGCS({ format: 'JSON' }, callback); - var pendingState = { status: { state: 'PENDING' } }; - example.mocks.job.getMetadata = sinon.stub().callsArgWith(0, null, pendingState); - example.program.pollExportJob(jobId, callback); - - assert(example.mocks.bigquery.job.calledOnce, 'bigquery.job called once'); - assert(example.mocks.job.getMetadata.calledOnce, 'getMetadata called once'); assert(callback.calledOnce, 'callback called once'); assert.equal(callback.firstCall.args.length, 1, 'callback received 1 argument'); - assert.deepEqual(callback.firstCall.args[0], new Error('Job %s is not done'), - 'callback received error' - ); - - assert(console.log.calledWith('PollExportJob: job status: %s', pendingState.status.state), - 'job status was reported' - ); - - var doneState = { status: { state: 'DONE' } }; - example.mocks.job.getMetadata = sinon.stub().callsArgWith(0, null, doneState); - example.program.pollExportJob(jobId, callback); - - assert(example.mocks.bigquery.job.calledTwice, 'bigquery.job called a second time'); - assert(example.mocks.job.getMetadata.calledOnce, 'new getMetadata called once'); - assert(callback.calledTwice, 'callback called a second time'); - - assert.equal(callback.secondCall.args.length, 2, 'callback received 2 arguments'); - assert.ifError(callback.secondCall.args[0], 'callback did not receive error'); - assert.deepEqual(callback.secondCall.args[1], example.mocks.metadata, - 'callback received metadata' - ); - - assert(console.log.calledWith('PollExportJob: job status: %s', doneState.status.state), - 'job status was reported' - ); - }); - - it('should handle error', function () { - var error = new Error('pollExportJobError'); - var example = getSample(); - var callback = sinon.stub(); - example.mocks.job.getMetadata = sinon.stub().callsArgWith(0, error); - example.program.pollExportJob(jobId, callback); - - assert(example.mocks.bigquery.job.calledOnce, 'bigquery.job called once'); - assert(example.mocks.job.getMetadata.calledOnce, 'getMetadata called once'); - assert(callback.calledOnce, 'callback called'); - assert.equal(callback.firstCall.args.length, 1, 'callback received 1 argument'); assert.equal(callback.firstCall.args[0], error, 'callback received error'); }); }); @@ -189,14 +128,5 @@ describe('bigquery:tables', function () { program.main(['export', 'bucket', 'file', 'dataset', 'table', 'JSON']); assert(program.exportTableToGCS.calledOnce); }); - - it('should call pollExportJob', function () { - var program = getSample().program; - program.pollExportJob = sinon.stub(); - var jobId = 'ABCDE'; - - program.main(['poll', jobId]); - assert(program.pollExportJob.calledWith(jobId)); - }); }); }); From 8f333df2e8a490aadafcb9240e4dc78c13b2dbb6 Mon Sep 17 00:00:00 2001 From: Ace Nassri Date: Tue, 23 Aug 2016 18:57:08 -0700 Subject: [PATCH 07/12] Remove magic strings --- bigquery/test/tables.test.js | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/bigquery/test/tables.test.js b/bigquery/test/tables.test.js index b10af95ee8..a0d91b8023 100644 --- a/bigquery/test/tables.test.js +++ b/bigquery/test/tables.test.js @@ -14,6 +14,11 @@ 'use strict'; var proxyquire = require('proxyquire').noCallThru(); +var bucket = 'bucket'; +var file = 'file'; +var dataset = 'dataset'; +var table = 'table'; +var format = 'JSON'; function getSample () { var bucketMock = { @@ -28,7 +33,6 @@ function getSample () { getMetadata: sinon.stub().callsArgWith(0, null, metadataMock), on: sinon.stub() }; - var tableMock = { export: sinon.stub().callsArgWith(2, null, jobMock) }; @@ -67,16 +71,15 @@ describe('bigquery:tables', function () { it('should export to a table', function () { var example = getSample(); var options = { - bucket: 'bucket', - file: 'file', - dataset: 'dataset', - table: 'table', - format: 'JSON', + bucket: bucket, + file: file, + dataset: dataset, + table: table, + format: format, gzip: true }; var callback = sinon.stub(); example.mocks.job.on.withArgs('complete').callsArgWith(1, example.mocks.metadata); - example.mocks.job.on.withArgs('error').returns(0); example.program.exportTableToGCS(options, callback); @@ -100,7 +103,7 @@ describe('bigquery:tables', function () { var example = getSample(); var callback = sinon.stub(); example.mocks.table.export = sinon.stub().callsArgWith(2, error); - example.program.exportTableToGCS({ format: 'JSON' }, callback); + example.program.exportTableToGCS({ format: format }, callback); assert(callback.calledOnce, 'callback called once'); assert.equal(callback.firstCall.args.length, 1, 'callback received 1 argument'); @@ -112,7 +115,7 @@ describe('bigquery:tables', function () { var example = getSample(); var callback = sinon.stub(); example.mocks.job.on.withArgs('error').callsArgWith(1, error); - example.program.exportTableToGCS({ format: 'JSON' }, callback); + example.program.exportTableToGCS({ format: format }, callback); assert(callback.calledOnce, 'callback called once'); assert.equal(callback.firstCall.args.length, 1, 'callback received 1 argument'); @@ -125,7 +128,7 @@ describe('bigquery:tables', function () { var program = getSample().program; program.exportTableToGCS = sinon.stub(); - program.main(['export', 'bucket', 'file', 'dataset', 'table', 'JSON']); + program.main(['export', bucket, file, dataset, table, format]); assert(program.exportTableToGCS.calledOnce); }); }); From 5b510a7f8121536db82fb0545884dec2c49f4b84 Mon Sep 17 00:00:00 2001 From: Ace Nassri Date: Wed, 24 Aug 2016 11:25:39 -0700 Subject: [PATCH 08/12] Minor tweaks + Add command-parsing tests --- bigquery/tables.js | 10 ++++++++-- bigquery/test/tables.test.js | 30 ++++++++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/bigquery/tables.js b/bigquery/tables.js index b26df361f5..f94f974a02 100644 --- a/bigquery/tables.js +++ b/bigquery/tables.js @@ -51,7 +51,6 @@ var storage = Storage(); */ function exportTableToGCS (options, callback) { var gcsFileObj = storage.bucket(options.bucket).file(options.file); - // Export table // See https://googlecloudplatform.github.io/gcloud-node/#/docs/google-cloud/latest/bigquery/table?method=export var table = bigquery.dataset(options.dataset).table(options.table); @@ -107,7 +106,14 @@ cli }, function (options) { program.exportTableToGCS(options, console.log); }) - .example('node $0 export sample-bigquery-export data.json github_samples natality JSON --gzip', 'Export github_samples:natality to gcs://sample-bigquery-export/data.json as gzipped JSON') + .example( + 'node $0 export sample-bigquery-export data.json github_samples natality', + 'Export github_samples:natality to gcs://sample-bigquery-export/data.json as raw JSON' + ) + .example( + 'node $0 export sample-bigquery-export data.csv github_samples natality -f CSV --gzip', + 'Export github_samples:natality to gcs://sample-bigquery-export/data.csv as gzipped CSV' + ) .wrap(100) .recommendCommands() .epilogue('For more information, see https://cloud.google.com/bigquery/exporting-data-from-bigquery'); diff --git a/bigquery/test/tables.test.js b/bigquery/test/tables.test.js index a0d91b8023..0eeb9a70aa 100644 --- a/bigquery/test/tables.test.js +++ b/bigquery/test/tables.test.js @@ -128,8 +128,34 @@ describe('bigquery:tables', function () { var program = getSample().program; program.exportTableToGCS = sinon.stub(); - program.main(['export', bucket, file, dataset, table, format]); - assert(program.exportTableToGCS.calledOnce); + program.main(['export', bucket, file, dataset, table]); + assert(program.exportTableToGCS.calledOnce, 'exportTableToGCS called'); + }); + + it('should recognize --gzip flag', function () { + var program = getSample().program; + program.exportTableToGCS = sinon.stub(); + + program.main(['export', bucket, file, dataset, table, '--gzip']); + assert(program.exportTableToGCS.calledOnce, 'exportTableToGCS called once'); + + var firstArgs = program.exportTableToGCS.firstCall.args; + assert.equal(firstArgs.length, 2, 'exportTableToGCS received 2 arguments'); + assert(firstArgs[0], 'exportTableToGCS received options'); + assert(firstArgs[0].gzip, 'exportTableToGCS received gzip as True'); + }); + + it('should recognize --format flag', function () { + var program = getSample().program; + program.exportTableToGCS = sinon.stub(); + + program.main(['export', bucket, file, dataset, table, '--format', 'CSV']); + assert(program.exportTableToGCS.calledOnce, 'exportTableToGCS called once'); + + var firstArgs = program.exportTableToGCS.firstCall.args; + assert.equal(firstArgs.length, 2, 'exportTableToGCS received 2 arguments'); + assert(firstArgs[0], 'exportTableToGCS received options'); + assert.equal(firstArgs[0].format, 'CSV', 'exportTableToGCS received format as CSV'); }); }); }); From 1d76a8ba81baed5bbee8ca8398cd136a74c82382 Mon Sep 17 00:00:00 2001 From: Ace Nassri Date: Wed, 24 Aug 2016 12:45:26 -0700 Subject: [PATCH 09/12] Add default format --- bigquery/tables.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bigquery/tables.js b/bigquery/tables.js index f94f974a02..1bf9e6ed07 100644 --- a/bigquery/tables.js +++ b/bigquery/tables.js @@ -96,7 +96,8 @@ cli global: true, requiresArg: true, type: 'string', - choices: ['JSON', 'CSV', 'AVRO'] + choices: ['JSON', 'CSV', 'AVRO'], + default: 'JSON' }, gzip: { global: true, From bd3477829bc468cb9f4876e44b4efa74e470ef7c Mon Sep 17 00:00:00 2001 From: Ace Nassri Date: Wed, 24 Aug 2016 14:14:07 -0700 Subject: [PATCH 10/12] Address comments --- bigquery/system-test/tables.test.js | 24 ++++++++++---- bigquery/tables.js | 51 ++++++++++++++--------------- 2 files changed, 42 insertions(+), 33 deletions(-) diff --git a/bigquery/system-test/tables.test.js b/bigquery/system-test/tables.test.js index 8748ccb110..05857b06b1 100644 --- a/bigquery/system-test/tables.test.js +++ b/bigquery/system-test/tables.test.js @@ -24,10 +24,8 @@ var options = { dataset: generateUuid(), table: generateUuid() }; -var BigQuery = require('@google-cloud/bigquery'); -var bigquery = BigQuery(); -var Storage = require('@google-cloud/storage'); -var storage = Storage(); +var bigquery = require('@google-cloud/bigquery')(); +var storage = require('@google-cloud/storage')(); var file = storage.bucket(options.bucket).file(options.file); describe('bigquery:tables', function () { @@ -54,9 +52,23 @@ describe('bigquery:tables', function () { }); after(function (done) { // Delete table export - file.delete(function () { + file.delete(function (err) { + if (err) { + return done(err); + } // Delete testing dataset/table - bigquery.dataset(options.dataset).delete({ force: true }, done); + bigquery.dataset(options.dataset).delete({ force: true }, function (err) { + if (err) { + return done(err); + } + // Delete bucket + storage.bucket(options.bucket).deleteFiles({ force: true }, function (err) { + if (err) { + return done(err); + } + storage.bucket(options.bucket).delete(done); + }); + }); }); }); diff --git a/bigquery/tables.js b/bigquery/tables.js index 1bf9e6ed07..7a574c7753 100644 --- a/bigquery/tables.js +++ b/bigquery/tables.js @@ -51,29 +51,27 @@ var storage = Storage(); */ function exportTableToGCS (options, callback) { var gcsFileObj = storage.bucket(options.bucket).file(options.file); - // Export table - // See https://googlecloudplatform.github.io/gcloud-node/#/docs/google-cloud/latest/bigquery/table?method=export var table = bigquery.dataset(options.dataset).table(options.table); - table.export( - gcsFileObj, - { - format: options.format, - gzip: options.gzip - }, - function (err, job) { - if (err) { - return callback(err); - } - console.log('ExportTableToGCS: submitted job %s!', job.id); + var config = { + format: options.format, + gzip: options.gzip + }; - job.on('error', function (err) { - return callback(err); - }); - job.on('complete', function (job) { - return callback(null, job); - }); + // Export table + // See https://googlecloudplatform.github.io/gcloud-node/#/docs/google-cloud/latest/bigquery/table?method=export + table.export(gcsFileObj, config, function (err, job) { + if (err) { + return callback(err); } - ); + console.log('ExportTableToGCS: submitted job %s!', job.id); + + job.on('error', function (err) { + return callback(err); + }); + job.on('complete', function (job) { + return callback(null, job); + }); + }); } // [END export_table_to_gcs] // [END complete] @@ -96,8 +94,7 @@ cli global: true, requiresArg: true, type: 'string', - choices: ['JSON', 'CSV', 'AVRO'], - default: 'JSON' + choices: ['JSON', 'CSV', 'AVRO'] }, gzip: { global: true, @@ -108,16 +105,16 @@ cli program.exportTableToGCS(options, console.log); }) .example( - 'node $0 export sample-bigquery-export data.json github_samples natality', - 'Export github_samples:natality to gcs://sample-bigquery-export/data.json as raw JSON' + 'node $0 export my-bucket my-file my-dataset my-table', + 'Export my-dataset:my-table to gcs://my-bucket/my-file as raw JSON' ) .example( - 'node $0 export sample-bigquery-export data.csv github_samples natality -f CSV --gzip', - 'Export github_samples:natality to gcs://sample-bigquery-export/data.csv as gzipped CSV' + 'node $0 export my-bucket my-file my-dataset my-table -f CSV --gzip', + 'Export my-dataset:my-table to gcs://my-bucket/my-file as gzipped CSV' ) .wrap(100) .recommendCommands() - .epilogue('For more information, see https://cloud.google.com/bigquery/exporting-data-from-bigquery'); + .epilogue('For more information, see https://cloud.google.com/bigquery/docs'); if (module === require.main) { program.main(process.argv.slice(2)); From 7ec5b5ae145861391d1a053fcb693072566f0fec Mon Sep 17 00:00:00 2001 From: Ace Nassri Date: Wed, 24 Aug 2016 14:19:30 -0700 Subject: [PATCH 11/12] Fix comments --- bigquery/system-test/tables.test.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bigquery/system-test/tables.test.js b/bigquery/system-test/tables.test.js index 05857b06b1..e0085a3812 100644 --- a/bigquery/system-test/tables.test.js +++ b/bigquery/system-test/tables.test.js @@ -61,11 +61,12 @@ describe('bigquery:tables', function () { if (err) { return done(err); } - // Delete bucket + // Delete files in bucket storage.bucket(options.bucket).deleteFiles({ force: true }, function (err) { if (err) { return done(err); } + // Delete bucket storage.bucket(options.bucket).delete(done); }); }); From 81735290b49c42a9b502d40579b899fb13861b96 Mon Sep 17 00:00:00 2001 From: Ace Nassri Date: Wed, 24 Aug 2016 14:26:59 -0700 Subject: [PATCH 12/12] Fix system test cleanup logic --- bigquery/system-test/tables.test.js | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/bigquery/system-test/tables.test.js b/bigquery/system-test/tables.test.js index e0085a3812..17a089e321 100644 --- a/bigquery/system-test/tables.test.js +++ b/bigquery/system-test/tables.test.js @@ -51,24 +51,15 @@ describe('bigquery:tables', function () { }); }); after(function (done) { - // Delete table export - file.delete(function (err) { - if (err) { - return done(err); - } - // Delete testing dataset/table - bigquery.dataset(options.dataset).delete({ force: true }, function (err) { + // Delete testing dataset/table + bigquery.dataset(options.dataset).delete({ force: true }, function () { + // Delete files + storage.bucket(options.bucket).deleteFiles({ force: true }, function (err) { if (err) { return done(err); } - // Delete files in bucket - storage.bucket(options.bucket).deleteFiles({ force: true }, function (err) { - if (err) { - return done(err); - } - // Delete bucket - storage.bucket(options.bucket).delete(done); - }); + // Delete bucket + storage.bucket(options.bucket).delete(done); }); }); });