From a58a5fa74636f203f83a15d7bba56bd3ffae3acc Mon Sep 17 00:00:00 2001 From: "Benjamin E. Coe" Date: Tue, 3 Aug 2021 12:20:45 -0400 Subject: [PATCH 01/27] feat!: initial generation of nodejs-storage-transfer library (#2) --- storagetransfer/package.json | 22 ++++++++++++ storagetransfer/quickstart.js | 54 ++++++++++++++++++++++++++++++ storagetransfer/test/quickstart.js | 44 ++++++++++++++++++++++++ 3 files changed, 120 insertions(+) create mode 100644 storagetransfer/package.json create mode 100644 storagetransfer/quickstart.js create mode 100644 storagetransfer/test/quickstart.js diff --git a/storagetransfer/package.json b/storagetransfer/package.json new file mode 100644 index 0000000000..c41b75b8b2 --- /dev/null +++ b/storagetransfer/package.json @@ -0,0 +1,22 @@ +{ + "name": "nodejs-storage-transfer", + "private": true, + "license": "Apache-2.0", + "author": "Google LLC", + "engines": { + "node": ">=10" + }, + "files": [ + "*.js" + ], + "scripts": { + "test": "c8 mocha --timeout 600000 test/*.js" + }, + "dependencies": { + "@google-cloud/storage-transfer": "^0.1.0" + }, + "devDependencies": { + "c8": "^7.1.0", + "mocha": "^8.0.0" + } +} diff --git a/storagetransfer/quickstart.js b/storagetransfer/quickstart.js new file mode 100644 index 0000000000..b42bd8e0b1 --- /dev/null +++ b/storagetransfer/quickstart.js @@ -0,0 +1,54 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +'use strict'; + +async function main(projectId = 'my-project') { + // [START nodejs_storage_transfer_quickstart] + // Imports the Google Cloud client library + + // remove this line after package is released + const { + StorageTransferServiceClient, + } = require('@google-cloud/storage-transfer'); + + // TODO(developer): replace with your prefered project ID. + // const projectId = 'my-project' + + // Creates a client + const client = new StorageTransferServiceClient(); + + async function listTransferJobs() { + const iterable = client.listTransferJobsAsync({ + filter: JSON.stringify({ + projectId, + jobNames: ['transferJobs/*'], + }), + }); + for await (const response of iterable) { + // process response + console.info(response); + } + } + listTransferJobs(); + // [END nodejs_storage_transfer_quickstart] +} + +main(...process.argv.slice(2)).catch(err => { + console.error(err.message); + process.exitCode = 1; +}); +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); diff --git a/storagetransfer/test/quickstart.js b/storagetransfer/test/quickstart.js new file mode 100644 index 0000000000..667413decb --- /dev/null +++ b/storagetransfer/test/quickstart.js @@ -0,0 +1,44 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ** This file is automatically generated by gapic-generator-typescript. ** +// ** https://github.com/googleapis/gapic-generator-typescript ** +// ** All changes to this file may be overwritten. ** + +'use strict'; + +const path = require('path'); +const cp = require('child_process'); +const {before, describe, it} = require('mocha'); +const { + StorageTransferServiceClient, +} = require('@google-cloud/storage-transfer'); + +const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'}); + +const cwd = path.join(__dirname, '..'); + +const client = new StorageTransferServiceClient(); + +describe('Quickstart', () => { + //TODO: remove this if not using the projectId + let projectId; + + before(async () => { + projectId = await client.getProjectId(); + }); + + it('should run quickstart', async () => { + execSync(`node ./quickstart.js ${projectId}`, {cwd}); + }); +}); From 76763d336eabe02e5daf2b47e78ec62824927c2f Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 3 Aug 2021 09:31:12 -0700 Subject: [PATCH 02/27] chore: release 1.0.0 (#3) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- storagetransfer/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storagetransfer/package.json b/storagetransfer/package.json index c41b75b8b2..4983ae30ed 100644 --- a/storagetransfer/package.json +++ b/storagetransfer/package.json @@ -13,7 +13,7 @@ "test": "c8 mocha --timeout 600000 test/*.js" }, "dependencies": { - "@google-cloud/storage-transfer": "^0.1.0" + "@google-cloud/storage-transfer": "^1.0.0" }, "devDependencies": { "c8": "^7.1.0", From e4c5698d086c05f51094d713b6818d09c898ddca Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 17 Aug 2021 17:10:47 +0000 Subject: [PATCH 03/27] chore: release 1.0.1 (#8) :robot: I have created a release \*beep\* \*boop\* --- ### [1.0.1](https://www.github.com/googleapis/nodejs-storage-transfer/compare/v1.0.0...v1.0.1) (2021-08-17) ### Bug Fixes * **deps:** google-gax v2.24.1 ([#7](https://www.github.com/googleapis/nodejs-storage-transfer/issues/7)) ([e52a0be](https://www.github.com/googleapis/nodejs-storage-transfer/commit/e52a0befc6497c68ea5401e35f3c1c7de0402941)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- storagetransfer/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storagetransfer/package.json b/storagetransfer/package.json index 4983ae30ed..dc928020a0 100644 --- a/storagetransfer/package.json +++ b/storagetransfer/package.json @@ -13,7 +13,7 @@ "test": "c8 mocha --timeout 600000 test/*.js" }, "dependencies": { - "@google-cloud/storage-transfer": "^1.0.0" + "@google-cloud/storage-transfer": "^1.0.1" }, "devDependencies": { "c8": "^7.1.0", From 01aef9c49c3fab8dddd44c9e1b32dc8bac9f0cbd Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 23 Aug 2021 18:50:37 +0000 Subject: [PATCH 04/27] chore: release 1.1.0 (#10) :robot: I have created a release \*beep\* \*boop\* --- ## [1.1.0](https://www.github.com/googleapis/nodejs-storage-transfer/compare/v1.0.1...v1.1.0) (2021-08-23) ### Features * turns on self-signed JWT feature flag ([#9](https://www.github.com/googleapis/nodejs-storage-transfer/issues/9)) ([47dde37](https://www.github.com/googleapis/nodejs-storage-transfer/commit/47dde3725b8907d1f974dca83b778ccd7e312bef)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- storagetransfer/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storagetransfer/package.json b/storagetransfer/package.json index dc928020a0..3807033f73 100644 --- a/storagetransfer/package.json +++ b/storagetransfer/package.json @@ -13,7 +13,7 @@ "test": "c8 mocha --timeout 600000 test/*.js" }, "dependencies": { - "@google-cloud/storage-transfer": "^1.0.1" + "@google-cloud/storage-transfer": "^1.1.0" }, "devDependencies": { "c8": "^7.1.0", From b5333fbeed231dfda4426f1b955577c1d471874a Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Mon, 23 Aug 2021 21:27:44 +0200 Subject: [PATCH 05/27] chore(deps): update dependency mocha to v9 (#4) --- storagetransfer/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storagetransfer/package.json b/storagetransfer/package.json index 3807033f73..2063bac337 100644 --- a/storagetransfer/package.json +++ b/storagetransfer/package.json @@ -17,6 +17,6 @@ }, "devDependencies": { "c8": "^7.1.0", - "mocha": "^8.0.0" + "mocha": "^9.0.0" } } From 37806ed13333dffb2c92440f7f34b7fbdc422340 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Fri, 10 Sep 2021 10:54:55 -0700 Subject: [PATCH 06/27] chore: release 1.1.1 (#15) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- storagetransfer/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storagetransfer/package.json b/storagetransfer/package.json index 2063bac337..819bdf4395 100644 --- a/storagetransfer/package.json +++ b/storagetransfer/package.json @@ -13,7 +13,7 @@ "test": "c8 mocha --timeout 600000 test/*.js" }, "dependencies": { - "@google-cloud/storage-transfer": "^1.1.0" + "@google-cloud/storage-transfer": "^1.1.1" }, "devDependencies": { "c8": "^7.1.0", From 94fe9dea0989b58925cf32173c56ff577c6e5835 Mon Sep 17 00:00:00 2001 From: JesseLovelace <43148100+JesseLovelace@users.noreply.github.com> Date: Tue, 16 Nov 2021 14:19:49 -0800 Subject: [PATCH 07/27] Update the quickstart sample to be in line with other languages (#24) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update the quickstart sample to be in line with other languages * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * run linter * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * remove unnecessary dependency Co-authored-by: Owl Bot Co-authored-by: Justin Beckwith --- storagetransfer/package.json | 5 ++- storagetransfer/quickstart.js | 66 ++++++++++++++++++--------- storagetransfer/test/quickstart.js | 71 ++++++++++++++++++++++++++++-- 3 files changed, 117 insertions(+), 25 deletions(-) diff --git a/storagetransfer/package.json b/storagetransfer/package.json index 819bdf4395..9883d004d0 100644 --- a/storagetransfer/package.json +++ b/storagetransfer/package.json @@ -16,7 +16,10 @@ "@google-cloud/storage-transfer": "^1.1.1" }, "devDependencies": { + "chai": "^4.2.0", "c8": "^7.1.0", - "mocha": "^9.0.0" + "mocha": "^9.0.0", + "uuid": "^8.0.0", + "@google-cloud/storage": "^5.15.3" } } diff --git a/storagetransfer/quickstart.js b/storagetransfer/quickstart.js index b42bd8e0b1..48f84bf902 100644 --- a/storagetransfer/quickstart.js +++ b/storagetransfer/quickstart.js @@ -13,35 +13,61 @@ 'use strict'; -async function main(projectId = 'my-project') { - // [START nodejs_storage_transfer_quickstart] - // Imports the Google Cloud client library +async function main(projectId = 'my-project', gcsSourceBucket, gcsSinkBucket) { + // [START storagetransfer_quickstart] + + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // Your project id + // const projectId = 'my-project' + + // The ID of the GCS bucket to transfer data from + // const gcsSourceBucket = 'my-source-bucket' - // remove this line after package is released + // The ID of the GCS bucket to transfer data to + // const gcsSinkBucket = 'my-sink-bucket' + + // Imports the Google Cloud client library const { StorageTransferServiceClient, } = require('@google-cloud/storage-transfer'); - // TODO(developer): replace with your prefered project ID. - // const projectId = 'my-project' - // Creates a client const client = new StorageTransferServiceClient(); - async function listTransferJobs() { - const iterable = client.listTransferJobsAsync({ - filter: JSON.stringify({ - projectId, - jobNames: ['transferJobs/*'], - }), - }); - for await (const response of iterable) { - // process response - console.info(response); - } + async function quickstart() { + // Creates a request to transfer from the source bucket to + // the sink bucket + const createRequest = { + transferJob: { + projectId: projectId, + transferSpec: { + gcsDataSource: {bucketName: gcsSourceBucket}, + gcsDataSink: {bucketName: gcsSinkBucket}, + }, + status: 'ENABLED', + }, + }; + + // Runs the request and creates the job + const response = await client.createTransferJob(createRequest); + + const jobName = response[0].name; + + const runRequest = { + jobName: jobName, + projectId: projectId, + }; + await client.runTransferJob(runRequest); + + console.log( + `Created and ran a transfer job from ${gcsSourceBucket} to ${gcsSinkBucket} with name ${jobName}` + ); } - listTransferJobs(); - // [END nodejs_storage_transfer_quickstart] + + quickstart(); + // [END storagetransfer_quickstart] } main(...process.argv.slice(2)).catch(err => { diff --git a/storagetransfer/test/quickstart.js b/storagetransfer/test/quickstart.js index 667413decb..0d0b1de628 100644 --- a/storagetransfer/test/quickstart.js +++ b/storagetransfer/test/quickstart.js @@ -17,28 +17,91 @@ 'use strict'; +const {assert} = require('chai'); const path = require('path'); const cp = require('child_process'); -const {before, describe, it} = require('mocha'); +const {after, before, describe, it} = require('mocha'); +const {Storage} = require('@google-cloud/storage'); const { StorageTransferServiceClient, } = require('@google-cloud/storage-transfer'); +const {v4: uuidv4} = require('uuid'); const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'}); const cwd = path.join(__dirname, '..'); const client = new StorageTransferServiceClient(); +const storage = new Storage(); +const sourceBucketName = `nodejs-sts-source${uuidv4()}`; +const sinkBucketName = `nodejs-sts-sink${uuidv4()}`; +const sourceBucket = storage.bucket(sourceBucketName); +const sinkBucket = storage.bucket(sinkBucketName); + +after(async () => { + await sourceBucket.delete().catch(console.error); + await sinkBucket.delete().catch(console.error); +}); describe('Quickstart', () => { - //TODO: remove this if not using the projectId let projectId; before(async () => { - projectId = await client.getProjectId(); + projectId = await storage.getProjectId(); + await sourceBucket.create(); + await sinkBucket.create(); }); + async function grantStsPermissions(projectId, bucket) { + const request = {projectId: projectId}; + const serviceAccount = await client.getGoogleServiceAccount(request); + const email = serviceAccount[0].accountEmail; + + const objectViewer = 'roles/storage.objectViewer'; + const bucketReader = 'roles/storage.legacyBucketReader'; + const bucketWriter = 'roles/storage.legacyBucketWriter'; + const members = ['serviceAccount:' + email]; + + const [policy] = await bucket.iam.getPolicy({requestedPolicyVersion: 3}); + + policy.bindings.push({ + role: objectViewer, + members: members, + }); + + policy.bindings.push({ + role: bucketReader, + members: members, + }); + + policy.bindings.push({ + role: bucketWriter, + members: members, + }); + + await bucket.iam.setPolicy(policy); + } + it('should run quickstart', async () => { - execSync(`node ./quickstart.js ${projectId}`, {cwd}); + await grantStsPermissions(projectId, sourceBucket); + await grantStsPermissions(projectId, sinkBucket); + const output = execSync( + `node ./quickstart.js ${projectId} ${sourceBucketName} ${sinkBucketName}`, + {cwd} + ); + console.log(output); + assert.include(output, 'transferJobs/'); + + // If it ran successfully and a job was created, delete it to clean up + const transferJobName = output.match(/transferJobs.*/)[0]; + const deleteRequest = { + projectId: projectId, + jobName: transferJobName, + transferJob: { + name: transferJobName, + status: 'DELETED', + }, + }; + await client.updateTransferJob(deleteRequest); }); }); From 89abd100ddd0f4aee3c63a7cc9ed9cc1777e8fef Mon Sep 17 00:00:00 2001 From: Daniel Bankhead Date: Tue, 1 Mar 2022 17:59:52 -0800 Subject: [PATCH 08/27] feat: Increase STS Sample Coverage (#42) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test: Prepare for multiple samples and tests * feat: Increase STS Sample Coverage * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * style: lint * ci: Add `pre-samples-test.sh` for AWS integration testing * style: Streamline error handling Co-authored-by: Owl Bot --- storagetransfer/aws-request.js | 110 +++++++++++ .../check-latest-transfer-operation.js | 67 +++++++ .../get-transfer-job-with-retries.js | 79 ++++++++ storagetransfer/nearline-request.js | 113 ++++++++++++ storagetransfer/package.json | 9 +- storagetransfer/quickstart.js | 27 ++- storagetransfer/test/aws-request.test.js | 72 ++++++++ .../check-latest-transfer-operation.test.js | 59 ++++++ .../get-transfer-job-with-retries.test.js | 51 ++++++ storagetransfer/test/nearline-request.test.js | 61 ++++++ storagetransfer/test/quickstart.js | 107 ----------- storagetransfer/test/quickstart.test.js | 59 ++++++ storagetransfer/test/transfer-check.test.js | 59 ++++++ storagetransfer/test/utils/bucket.js | 173 ++++++++++++++++++ storagetransfer/test/utils/index.js | 27 +++ storagetransfer/test/utils/sample.js | 38 ++++ storagetransfer/test/utils/transfer.js | 110 +++++++++++ storagetransfer/transfer-check.js | 74 ++++++++ 18 files changed, 1170 insertions(+), 125 deletions(-) create mode 100644 storagetransfer/aws-request.js create mode 100644 storagetransfer/check-latest-transfer-operation.js create mode 100644 storagetransfer/get-transfer-job-with-retries.js create mode 100644 storagetransfer/nearline-request.js create mode 100644 storagetransfer/test/aws-request.test.js create mode 100644 storagetransfer/test/check-latest-transfer-operation.test.js create mode 100644 storagetransfer/test/get-transfer-job-with-retries.test.js create mode 100644 storagetransfer/test/nearline-request.test.js delete mode 100644 storagetransfer/test/quickstart.js create mode 100644 storagetransfer/test/quickstart.test.js create mode 100644 storagetransfer/test/transfer-check.test.js create mode 100644 storagetransfer/test/utils/bucket.js create mode 100644 storagetransfer/test/utils/index.js create mode 100644 storagetransfer/test/utils/sample.js create mode 100644 storagetransfer/test/utils/transfer.js create mode 100644 storagetransfer/transfer-check.js diff --git a/storagetransfer/aws-request.js b/storagetransfer/aws-request.js new file mode 100644 index 0000000000..296e120e99 --- /dev/null +++ b/storagetransfer/aws-request.js @@ -0,0 +1,110 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +async function main( + projectId, + description, + awsSourceBucket, + awsAccessKeyId, + awsSecretAccessKey, + gcsSinkBucket +) { + // [START storagetransfer_transfer_from_aws] + + // Imports the Google Cloud client library + const { + StorageTransferServiceClient, + } = require('@google-cloud/storage-transfer'); + + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // The ID of the Google Cloud Platform Project that owns the job + // projectId = 'my-project-id' + + // A useful description for your transfer job + // description = 'My transfer job' + + // AWS S3 source bucket name + // awsSourceBucket = 'my-s3-source-bucket' + + // AWS Access Key ID + // awsAccessKeyId = 'AKIA...' + + // AWS Secret Access Key + // awsSecretAccessKey = 'HEAoMK2.../...ku8' + + // Google Cloud Storage destination bucket name + // gcsSinkBucket = 'my-gcs-destination-bucket' + + // Creates a client + const client = new StorageTransferServiceClient(); + + /** + * Creates a one-time transfer job from Amazon S3 to Google Cloud Storage. + */ + async function transferFromS3() { + // Setting the start date and the end date as the same time creates a + // one-time transfer + const now = new Date(); + const oneTimeSchedule = { + day: now.getDate(), + month: now.getMonth() + 1, + year: now.getFullYear(), + }; + + // Runs the request and creates the job + const [transferJob] = await client.createTransferJob({ + transferJob: { + projectId, + description, + status: 'ENABLED', + schedule: { + scheduleStartDate: oneTimeSchedule, + scheduleEndDate: oneTimeSchedule, + }, + transferSpec: { + awsS3DataSource: { + bucketName: awsSourceBucket, + awsAccessKey: { + accessKeyId: awsAccessKeyId, + secretAccessKey: awsSecretAccessKey, + }, + }, + gcsDataSink: { + bucketName: gcsSinkBucket, + }, + }, + }, + }); + + console.log( + `Created and ran a transfer job from '${awsSourceBucket}' to '${gcsSinkBucket}' with name ${transferJob.name}` + ); + } + + transferFromS3(); + // [END storagetransfer_transfer_from_aws] +} + +main(...process.argv.slice(2)); + +process.on('unhandledRejection', err => { + console.error(err); + process.exitCode = 1; +}); diff --git a/storagetransfer/check-latest-transfer-operation.js b/storagetransfer/check-latest-transfer-operation.js new file mode 100644 index 0000000000..0d9b7f7b89 --- /dev/null +++ b/storagetransfer/check-latest-transfer-operation.js @@ -0,0 +1,67 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +async function main(projectId, jobName) { + // [START storagetransfer_get_latest_transfer_operation] + + // Imports the Google Cloud client library + const { + StorageTransferServiceClient, + } = require('@google-cloud/storage-transfer'); + + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // The ID of the Google Cloud Platform Project that owns the job + // projectId = 'my-project-id' + + // Storage Transfer Service job name + // jobName = 'transferJobs/1234567890' + + // Creates a client + const client = new StorageTransferServiceClient(); + + /** + * Checks the latest transfer operation for a given transfer job. + */ + async function checkLatestTransferOperation() { + const [transferJob] = await client.getTransferJob({projectId, jobName}); + + if (transferJob.latestOperationName) { + const [transferOperation] = await client.operationsClient.getOperation({ + name: transferJob.latestOperationName, + }); + + const operation = JSON.stringify(transferOperation, null, 2); + + console.log(`Latest transfer operation for '${jobName}': ${operation}`); + } else { + console.log(`Transfer job '${jobName}' has not ran yet.`); + } + } + + checkLatestTransferOperation(); + // [END storagetransfer_get_latest_transfer_operation] +} + +main(...process.argv.slice(2)); + +process.on('unhandledRejection', err => { + console.error(err); + process.exitCode = 1; +}); diff --git a/storagetransfer/get-transfer-job-with-retries.js b/storagetransfer/get-transfer-job-with-retries.js new file mode 100644 index 0000000000..0d15aa07f8 --- /dev/null +++ b/storagetransfer/get-transfer-job-with-retries.js @@ -0,0 +1,79 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +async function main(projectId, jobName, maxRetryDelayMillis) { + // [START storagetransfer_create_retry_handler] + + // Imports the Google Cloud client library + const { + StorageTransferServiceClient, + } = require('@google-cloud/storage-transfer'); + + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // The ID of the Google Cloud Platform Project that owns the job + // projectId = 'my-project-id' + + // Storage Transfer Service job name + // jobName = 'transferJobs/1234567890' + + // The maximum delay time, in milliseconds, between requests + // maxRetryDelayMillis = 60000 + + // Creates a client + const client = new StorageTransferServiceClient(); + + /** + * Check the latest transfer operation associated with a transfer job + * with retries. + */ + async function getTransferJobWithRetries() { + // Setting the start date and the end date as the same time creates a + // one-time transfer + + const options = { + retry: { + backoffSettings: { + maxRetryDelayMillis, + }, + }, + }; + + const [transferJob] = await client.getTransferJob( + {projectId, jobName}, + options + ); + + console.log( + `Fetched transfer job: ${transferJob.name} with a maximum of ${maxRetryDelayMillis}ms delay time between requests` + ); + } + + getTransferJobWithRetries(); + // [END storagetransfer_create_retry_handler] +} + +const [projectId, jobName, maxRetryDelayMillis] = [...process.argv.slice(2)]; + +main(projectId, jobName, Number.parseInt(maxRetryDelayMillis)); + +process.on('unhandledRejection', err => { + console.error(err); + process.exitCode = 1; +}); diff --git a/storagetransfer/nearline-request.js b/storagetransfer/nearline-request.js new file mode 100644 index 0000000000..63b2e9c49c --- /dev/null +++ b/storagetransfer/nearline-request.js @@ -0,0 +1,113 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +async function main( + projectId, + description, + gcsSourceBucket, + gcsSinkBucket, + startDate = new Date() +) { + // [START storagetransfer_transfer_to_nearline] + + // Imports the Google Cloud client library + const { + StorageTransferServiceClient, + } = require('@google-cloud/storage-transfer'); + + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // The ID of the Google Cloud Platform Project that owns the job + // projectId = 'my-project-id' + + // A useful description for your transfer job + // description = 'My transfer job' + + // Google Cloud Storage source bucket name + // gcsSourceBucket = 'my-gcs-source-bucket' + + // Google Cloud Storage destination bucket name + // gcsSinkBucket = 'my-gcs-destination-bucket' + + // Date to start daily migration + // startDate = new Date() + + // Creates a client + const client = new StorageTransferServiceClient(); + + /** + * Create a daily migration from a GCS bucket to another GCS bucket for + * objects untouched for 30+ days. + */ + async function createDailyNearline30DayMigration() { + // Runs the request and creates the job + const [transferJob] = await client.createTransferJob({ + transferJob: { + projectId, + description, + status: 'ENABLED', + schedule: { + scheduleStartDate: { + day: startDate.getDate(), + month: startDate.getMonth() + 1, + year: startDate.getFullYear(), + }, + }, + transferSpec: { + gcsDataSource: { + bucketName: gcsSourceBucket, + }, + gcsDataSink: { + bucketName: gcsSinkBucket, + }, + objectConditions: { + minTimeElapsedSinceLastModification: { + seconds: 2592000, // 30 days + }, + }, + transferOptions: { + deleteObjectsFromSourceAfterTransfer: true, + }, + }, + }, + }); + + console.log(`Created transferJob: ${transferJob.name}`); + } + + createDailyNearline30DayMigration(); + // [END storagetransfer_transfer_to_nearline] +} + +const [projectId, description, gcsSourceBucket, gcsSinkBucket, startDate] = [ + ...process.argv.slice(2), +]; + +main( + projectId, + description, + gcsSourceBucket, + gcsSinkBucket, + new Date(startDate) +); + +process.on('unhandledRejection', err => { + console.error(err); + process.exitCode = 1; +}); diff --git a/storagetransfer/package.json b/storagetransfer/package.json index 9883d004d0..470f7a8ba8 100644 --- a/storagetransfer/package.json +++ b/storagetransfer/package.json @@ -10,16 +10,17 @@ "*.js" ], "scripts": { - "test": "c8 mocha --timeout 600000 test/*.js" + "test": "c8 mocha ---parallel --timeout 600000 test/*.test.js" }, "dependencies": { "@google-cloud/storage-transfer": "^1.1.1" }, "devDependencies": { - "chai": "^4.2.0", + "@google-cloud/storage": "^5.15.3", + "aws-sdk": "^2.1073.0", "c8": "^7.1.0", + "chai": "^4.2.0", "mocha": "^9.0.0", - "uuid": "^8.0.0", - "@google-cloud/storage": "^5.15.3" + "uuid": "^8.0.0" } } diff --git a/storagetransfer/quickstart.js b/storagetransfer/quickstart.js index 48f84bf902..f8d689408a 100644 --- a/storagetransfer/quickstart.js +++ b/storagetransfer/quickstart.js @@ -16,6 +16,11 @@ async function main(projectId = 'my-project', gcsSourceBucket, gcsSinkBucket) { // [START storagetransfer_quickstart] + // Imports the Google Cloud client library + const { + StorageTransferServiceClient, + } = require('@google-cloud/storage-transfer'); + /** * TODO(developer): Uncomment the following lines before running the sample. */ @@ -28,14 +33,12 @@ async function main(projectId = 'my-project', gcsSourceBucket, gcsSinkBucket) { // The ID of the GCS bucket to transfer data to // const gcsSinkBucket = 'my-sink-bucket' - // Imports the Google Cloud client library - const { - StorageTransferServiceClient, - } = require('@google-cloud/storage-transfer'); - // Creates a client const client = new StorageTransferServiceClient(); + /** + * Creates a one-time transfer job. + */ async function quickstart() { // Creates a request to transfer from the source bucket to // the sink bucket @@ -51,18 +54,16 @@ async function main(projectId = 'my-project', gcsSourceBucket, gcsSinkBucket) { }; // Runs the request and creates the job - const response = await client.createTransferJob(createRequest); - - const jobName = response[0].name; + const [transferJob] = await client.createTransferJob(createRequest); const runRequest = { - jobName: jobName, + jobName: transferJob.name, projectId: projectId, }; await client.runTransferJob(runRequest); console.log( - `Created and ran a transfer job from ${gcsSourceBucket} to ${gcsSinkBucket} with name ${jobName}` + `Created and ran a transfer job from ${gcsSourceBucket} to ${gcsSinkBucket} with name ${transferJob.name}` ); } @@ -70,10 +71,8 @@ async function main(projectId = 'my-project', gcsSourceBucket, gcsSinkBucket) { // [END storagetransfer_quickstart] } -main(...process.argv.slice(2)).catch(err => { - console.error(err.message); - process.exitCode = 1; -}); +main(...process.argv.slice(2)); + process.on('unhandledRejection', err => { console.error(err.message); process.exitCode = 1; diff --git a/storagetransfer/test/aws-request.test.js b/storagetransfer/test/aws-request.test.js new file mode 100644 index 0000000000..ce5e37e720 --- /dev/null +++ b/storagetransfer/test/aws-request.test.js @@ -0,0 +1,72 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const {assert} = require('chai'); +const {after, before, describe, it} = require('mocha'); + +const {BucketManager, TransferJobManager, runSample} = require('./utils'); + +describe('aws-request', () => { + const testBucketManager = new BucketManager(); + const testTransferJobManager = new TransferJobManager(); + + let projectId; + let description; + let awsSourceBucket; + let awsAccessKeyId; + let awsSecretAccessKey; + let gcsSinkBucket; + + before(async () => { + awsAccessKeyId = process.env.AWS_ACCESS_KEY_ID; + awsSecretAccessKey = process.env.AWS_SECRET_ACCESS_KEY; + + assert( + awsAccessKeyId && awsSecretAccessKey, + 'environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are required' + ); + + projectId = await testBucketManager.getProjectId(); + awsSourceBucket = await testBucketManager.generateS3Bucket(); + gcsSinkBucket = (await testBucketManager.generateGCSBucket()).name; + description = `My transfer job from '${awsSourceBucket}' -> '${gcsSinkBucket}'`; + }); + + after(async () => { + await testBucketManager.deleteBuckets(); + await testTransferJobManager.cleanUp(); + }); + + it('should create a transfer job from S3 to GCS', async () => { + const output = await runSample('aws-request', [ + projectId, + description, + awsSourceBucket, + awsAccessKeyId, + awsSecretAccessKey, + gcsSinkBucket, + ]); + + assert.include(output, 'Created and ran a transfer job'); + + // If it ran successfully and a job was created, delete it to clean up + const [jobName] = output.match(/transferJobs.*/); + + testTransferJobManager.transferJobToCleanUp(jobName); + }); +}); diff --git a/storagetransfer/test/check-latest-transfer-operation.test.js b/storagetransfer/test/check-latest-transfer-operation.test.js new file mode 100644 index 0000000000..c0e54c477f --- /dev/null +++ b/storagetransfer/test/check-latest-transfer-operation.test.js @@ -0,0 +1,59 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const {assert} = require('chai'); +const {after, before, describe, it} = require('mocha'); + +const {BucketManager, TransferJobManager, runSample} = require('./utils'); + +describe('check-latest-transfer-operation', () => { + const testBucketManager = new BucketManager(); + const testTransferJobManager = new TransferJobManager(); + + let projectId; + let transferJob; + + before(async () => { + projectId = await testBucketManager.getProjectId(); + const result = await testTransferJobManager.createTestTransferJob(); + + transferJob = result.transferJob; + }); + + after(async () => { + await testBucketManager.deleteBuckets(); + await testTransferJobManager.cleanUp(); + }); + + it('should check latest transfer operation', async () => { + await testTransferJobManager.client.runTransferJob({ + projectId, + jobName: transferJob.name, + }); + + const output = await runSample('check-latest-transfer-operation', [ + projectId, + transferJob.name, + ]); + + const formattedTransferJob = transferJob.name.replace('/', '-'); + + // Find the transfer operation from the transfer job in the output + assert.include(output, `transferOperations/${formattedTransferJob}-`); + }); +}); diff --git a/storagetransfer/test/get-transfer-job-with-retries.test.js b/storagetransfer/test/get-transfer-job-with-retries.test.js new file mode 100644 index 0000000000..51a011c157 --- /dev/null +++ b/storagetransfer/test/get-transfer-job-with-retries.test.js @@ -0,0 +1,51 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const {assert} = require('chai'); +const {after, before, describe, it} = require('mocha'); + +const {BucketManager, TransferJobManager, runSample} = require('./utils'); + +describe('get-transfer-job-with-retries', () => { + const testBucketManager = new BucketManager(); + const testTransferJobManager = new TransferJobManager(); + + let projectId; + let transferJob; + + before(async () => { + projectId = await testBucketManager.getProjectId(); + const result = await testTransferJobManager.createTestTransferJob(); + + transferJob = result.transferJob; + }); + + after(async () => { + await testBucketManager.deleteBuckets(); + await testTransferJobManager.cleanUp(); + }); + + it('should get a transfer job with retries', async () => { + const output = await runSample('get-transfer-job-with-retries', [ + projectId, + transferJob.name, + ]); + + assert.include(output, `Fetched transfer job: ${transferJob.name}`); + }); +}); diff --git a/storagetransfer/test/nearline-request.test.js b/storagetransfer/test/nearline-request.test.js new file mode 100644 index 0000000000..f730ced7c1 --- /dev/null +++ b/storagetransfer/test/nearline-request.test.js @@ -0,0 +1,61 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const {assert} = require('chai'); +const {after, before, describe, it} = require('mocha'); + +const {BucketManager, TransferJobManager, runSample} = require('./utils'); + +describe('nearline-request', () => { + const testBucketManager = new BucketManager(); + const testTransferJobManager = new TransferJobManager(); + + let projectId; + let description; + let gcsSourceBucket; + let gcsSinkBucket; + + before(async () => { + projectId = await testBucketManager.getProjectId(); + gcsSourceBucket = (await testBucketManager.generateGCSBucket()).name; + gcsSinkBucket = (await testBucketManager.generateGCSBucket()).name; + description = `My transfer job from '${gcsSourceBucket}' -> '${gcsSinkBucket}'`; + }); + + after(async () => { + await testBucketManager.deleteBuckets(); + await testTransferJobManager.cleanUp(); + }); + + it('should create a daily transfer job from S3 to GCS', async () => { + const output = await runSample('nearline-request', [ + projectId, + description, + gcsSourceBucket, + gcsSinkBucket, + new Date().toISOString(), + ]); + + assert.include(output, 'Created transferJob: transferJobs/'); + + // If it ran successfully and a job was created, delete it to clean up + const [jobName] = output.match(/transferJobs.*/); + + testTransferJobManager.transferJobToCleanUp(jobName); + }); +}); diff --git a/storagetransfer/test/quickstart.js b/storagetransfer/test/quickstart.js deleted file mode 100644 index 0d0b1de628..0000000000 --- a/storagetransfer/test/quickstart.js +++ /dev/null @@ -1,107 +0,0 @@ -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// ** This file is automatically generated by gapic-generator-typescript. ** -// ** https://github.com/googleapis/gapic-generator-typescript ** -// ** All changes to this file may be overwritten. ** - -'use strict'; - -const {assert} = require('chai'); -const path = require('path'); -const cp = require('child_process'); -const {after, before, describe, it} = require('mocha'); -const {Storage} = require('@google-cloud/storage'); -const { - StorageTransferServiceClient, -} = require('@google-cloud/storage-transfer'); -const {v4: uuidv4} = require('uuid'); - -const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'}); - -const cwd = path.join(__dirname, '..'); - -const client = new StorageTransferServiceClient(); -const storage = new Storage(); -const sourceBucketName = `nodejs-sts-source${uuidv4()}`; -const sinkBucketName = `nodejs-sts-sink${uuidv4()}`; -const sourceBucket = storage.bucket(sourceBucketName); -const sinkBucket = storage.bucket(sinkBucketName); - -after(async () => { - await sourceBucket.delete().catch(console.error); - await sinkBucket.delete().catch(console.error); -}); - -describe('Quickstart', () => { - let projectId; - - before(async () => { - projectId = await storage.getProjectId(); - await sourceBucket.create(); - await sinkBucket.create(); - }); - - async function grantStsPermissions(projectId, bucket) { - const request = {projectId: projectId}; - const serviceAccount = await client.getGoogleServiceAccount(request); - const email = serviceAccount[0].accountEmail; - - const objectViewer = 'roles/storage.objectViewer'; - const bucketReader = 'roles/storage.legacyBucketReader'; - const bucketWriter = 'roles/storage.legacyBucketWriter'; - const members = ['serviceAccount:' + email]; - - const [policy] = await bucket.iam.getPolicy({requestedPolicyVersion: 3}); - - policy.bindings.push({ - role: objectViewer, - members: members, - }); - - policy.bindings.push({ - role: bucketReader, - members: members, - }); - - policy.bindings.push({ - role: bucketWriter, - members: members, - }); - - await bucket.iam.setPolicy(policy); - } - - it('should run quickstart', async () => { - await grantStsPermissions(projectId, sourceBucket); - await grantStsPermissions(projectId, sinkBucket); - const output = execSync( - `node ./quickstart.js ${projectId} ${sourceBucketName} ${sinkBucketName}`, - {cwd} - ); - console.log(output); - assert.include(output, 'transferJobs/'); - - // If it ran successfully and a job was created, delete it to clean up - const transferJobName = output.match(/transferJobs.*/)[0]; - const deleteRequest = { - projectId: projectId, - jobName: transferJobName, - transferJob: { - name: transferJobName, - status: 'DELETED', - }, - }; - await client.updateTransferJob(deleteRequest); - }); -}); diff --git a/storagetransfer/test/quickstart.test.js b/storagetransfer/test/quickstart.test.js new file mode 100644 index 0000000000..9c8a5a1b27 --- /dev/null +++ b/storagetransfer/test/quickstart.test.js @@ -0,0 +1,59 @@ +// Copyright 2021 Google LLC + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ** This file is automatically generated by gapic-generator-typescript. ** +// ** https://github.com/googleapis/gapic-generator-typescript ** +// ** All changes to this file may be overwritten. ** + +'use strict'; + +const {assert} = require('chai'); +const {after, before, describe, it} = require('mocha'); + +const {BucketManager, TransferJobManager, runSample} = require('./utils'); + +describe('quickstart', () => { + const testBucketManager = new BucketManager(); + const testTransferJobManager = new TransferJobManager(); + + let projectId; + let sourceBucket; + let sinkBucket; + + before(async () => { + projectId = await testBucketManager.getProjectId(); + sourceBucket = (await testBucketManager.generateGCSBucket()).name; + sinkBucket = (await testBucketManager.generateGCSBucket()).name; + }); + + after(async () => { + await testBucketManager.deleteBuckets(); + await testTransferJobManager.cleanUp(); + }); + + it('should run quickstart', async () => { + const output = await runSample('quickstart', [ + projectId, + sourceBucket, + sinkBucket, + ]); + + assert.include(output, 'transferJobs/'); + + // If it ran successfully and a job was created, delete it to clean up + const [jobName] = output.match(/transferJobs.*/); + + testTransferJobManager.transferJobToCleanUp(jobName); + }); +}); diff --git a/storagetransfer/test/transfer-check.test.js b/storagetransfer/test/transfer-check.test.js new file mode 100644 index 0000000000..3f59ef54f0 --- /dev/null +++ b/storagetransfer/test/transfer-check.test.js @@ -0,0 +1,59 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const {assert} = require('chai'); +const {after, before, describe, it} = require('mocha'); + +const {BucketManager, TransferJobManager, runSample} = require('./utils'); + +describe('transfer-check', () => { + const testBucketManager = new BucketManager(); + const testTransferJobManager = new TransferJobManager(); + + let projectId; + let transferJob; + + before(async () => { + projectId = await testBucketManager.getProjectId(); + const result = await testTransferJobManager.createTestTransferJob(); + + transferJob = result.transferJob; + }); + + after(async () => { + await testBucketManager.deleteBuckets(); + await testTransferJobManager.cleanUp(); + }); + + it('should list operations for a transfer job', async () => { + await testTransferJobManager.client.runTransferJob({ + projectId, + jobName: transferJob.name, + }); + + const output = await runSample('transfer-check', [ + projectId, + transferJob.name, + ]); + + const formattedTransferJob = transferJob.name.replace('/', '-'); + + // Find at least 1 transfer operation from the transfer job in the output + assert.include(output, `transferOperations/${formattedTransferJob}-`); + }); +}); diff --git a/storagetransfer/test/utils/bucket.js b/storagetransfer/test/utils/bucket.js new file mode 100644 index 0000000000..4a6c26567b --- /dev/null +++ b/storagetransfer/test/utils/bucket.js @@ -0,0 +1,173 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +// eslint-disable-next-line no-unused-vars +const {Storage, Bucket} = require('@google-cloud/storage'); +const { + StorageTransferServiceClient, +} = require('@google-cloud/storage-transfer'); +const AWS = require('aws-sdk'); +const uuid = require('uuid'); + +class BucketManager { + constructor() { + this.client = new StorageTransferServiceClient(); + this.s3 = new AWS.S3({apiVersion: '2006-03-01'}); + this.storage = new Storage(); + + /** + * The GCP Project ID. Cached after initial request + */ + this._cachedProjectId = ''; + + /** + * @type {Bucket[]} + */ + this.gcsBuckets = []; + /** + * @type {string[]} + */ + this.s3Buckets = []; + } + + async getProjectId() { + if (!this._cachedProjectId) { + this._cachedProjectId = await this.storage.getProjectId(); + } + + return this._cachedProjectId; + } + + /** + * Generates a unique name for GCS and S3 buckets. + * + * @returns {string} Name of bucket + */ + static generateBucketName() { + return `nodejs-sts-samples-${uuid.v4()}`; + } + + /** + * Configures permissions for STS to read/write to the bucket. + * + * @param {Bucket} bucket + */ + async grantSTSPermissions(bucket) { + const [serviceAccount] = await this.client.getGoogleServiceAccount({ + projectId: await this.getProjectId(), + }); + const email = serviceAccount.accountEmail; + + const objectViewer = 'roles/storage.objectViewer'; + const bucketReader = 'roles/storage.legacyBucketReader'; + const bucketWriter = 'roles/storage.legacyBucketWriter'; + const members = [`serviceAccount:${email}`]; + + const [policy] = await bucket.iam.getPolicy({requestedPolicyVersion: 3}); + + policy.bindings.push({ + role: objectViewer, + members: members, + }); + + policy.bindings.push({ + role: bucketReader, + members: members, + }); + + policy.bindings.push({ + role: bucketWriter, + members: members, + }); + + await bucket.iam.setPolicy(policy); + } + + /** + * Generates a unique GCS bucket for testing. + * Configures STS read/write perms on the bucket. + * + * Is cached for easy clean-up via {#deleteBuckets}. + * + * @returns + */ + async generateGCSBucket() { + const name = await BucketManager.generateBucketName(); + const bucket = this.storage.bucket(name); + this.gcsBuckets.push(bucket); + + await bucket.create(); + await this.grantSTSPermissions(bucket); + + return bucket; + } + + /** + * Generates a unique GCS bucket for testing. + * Configures STS read/write perms on the bucket. + * + * Is cached for easy clean-up via {#deleteBuckets}. + * + * @returns + */ + + async generateS3Bucket() { + const name = await BucketManager.generateBucketName(); + + await new Promise((resolve, reject) => { + this.s3.createBucket({Bucket: name}, (error, data) => { + if (error) return reject(error); + + resolve(data); + }); + }); + + this.s3Buckets.push(name); + + return name; + } + + /** + * Deletes generated GCS & S3 test buckets. + */ + async deleteBuckets() { + for (const bucket of this.gcsBuckets) { + try { + await bucket.delete(); + } catch (e) { + console.error(e); + } + } + + for (const bucket of this.s3Buckets) { + try { + await new Promise((resolve, reject) => { + this.s3.deleteBucket({Bucket: bucket}, (error, data) => { + if (error) return reject(error); + + resolve(data); + }); + }); + } catch (e) { + console.error(e); + } + } + } +} + +module.exports = {BucketManager}; diff --git a/storagetransfer/test/utils/index.js b/storagetransfer/test/utils/index.js new file mode 100644 index 0000000000..f388c16907 --- /dev/null +++ b/storagetransfer/test/utils/index.js @@ -0,0 +1,27 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const bucket = require('./bucket'); +const sample = require('./sample'); +const transfer = require('./transfer'); + +module.exports = { + ...bucket, + ...sample, + ...transfer, +}; diff --git a/storagetransfer/test/utils/sample.js b/storagetransfer/test/utils/sample.js new file mode 100644 index 0000000000..5b891c91cb --- /dev/null +++ b/storagetransfer/test/utils/sample.js @@ -0,0 +1,38 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const {execFileSync} = require('child_process'); +const path = require('path'); + +/** + * Runs a sample and returns its output + * + * @param {string} sample name of sample to run. No need for '.js' suffix. + * @param {string[]} args the arguments to pass to the sample + * @returns {string} output of the command + */ +async function runSample(sample, args = []) { + return execFileSync('node', [`${sample}.js`, ...args], { + encoding: 'utf-8', + cwd: path.join(__dirname, '..', '..'), + }); +} + +module.exports = { + runSample, +}; diff --git a/storagetransfer/test/utils/transfer.js b/storagetransfer/test/utils/transfer.js new file mode 100644 index 0000000000..f7c1a767ec --- /dev/null +++ b/storagetransfer/test/utils/transfer.js @@ -0,0 +1,110 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const { + StorageTransferServiceClient, +} = require('@google-cloud/storage-transfer'); + +const {BucketManager} = require('./bucket'); + +class TransferJobManager { + constructor() { + this.bucketManager = new BucketManager(); + this.client = new StorageTransferServiceClient(); + + /** + * List of transferJobs to delete + * + * @type {string[]} + */ + this.transferJobsToCleanup = []; + } + + /** + * Clean up transfer jobs used for testing. + */ + async cleanUp() { + for (const jobName of this.transferJobsToCleanup) { + await this.deleteTransferJob(jobName); + } + + await this.bucketManager.deleteBuckets(); + } + + /** + * Creates a transfer job for testing, including test buckets. + * The job generated is not ran automatically. + * The source bucket and destination buckets do not contain objects. + * + * The job is automatically added for queue for easy clean-up via {#cleanUp}. + */ + async createTestTransferJob() { + const sourceBucket = await this.bucketManager.generateGCSBucket(); + const destBucket = await this.bucketManager.generateGCSBucket(); + + const [transferJob] = await this.client.createTransferJob({ + transferJob: { + projectId: await this.client.getProjectId(), + transferSpec: { + gcsDataSource: { + bucketName: sourceBucket.name, + }, + gcsDataSink: { + bucketName: destBucket.name, + }, + }, + status: 'ENABLED', + }, + }); + + this.transferJobToCleanUp(transferJob.name); + + return { + sourceBucket, + destBucket, + transferJob, + }; + } + + /** + * Deletes a STS transfer job. + * + * @param {string} jobName the name of the STS Job to delete + */ + async deleteTransferJob(jobName) { + await this.client.updateTransferJob({ + projectId: await this.client.getProjectId(), + jobName, + transferJob: { + name: jobName, + status: 'DELETED', + }, + }); + } + + /** + * Adds Transfer Job to queue for easy clean-up via {#cleanUp}. + * + * @param {string} jobName the name of the STS Job to queue + */ + transferJobToCleanUp(jobName) { + this.transferJobsToCleanup.push(jobName); + } +} + +module.exports = {TransferJobManager}; diff --git a/storagetransfer/transfer-check.js b/storagetransfer/transfer-check.js new file mode 100644 index 0000000000..6e68ff64d7 --- /dev/null +++ b/storagetransfer/transfer-check.js @@ -0,0 +1,74 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +async function main(projectId, jobName) { + // [START storagetransfer_transfer_check] + + // Imports the Google Cloud client library + const { + StorageTransferServiceClient, + protos, + } = require('@google-cloud/storage-transfer'); + + // Proto for TransferOperation + const TransferOperation = protos.google.storagetransfer.v1.TransferOperation; + + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // The ID of the Google Cloud Platform Project that owns the job + // projectId = 'my-project-id' + + // Storage Transfer Service job name + // jobName = 'transferJobs/1234567890' + + // Creates a client + const client = new StorageTransferServiceClient(); + + /** + * Lists operations for a transfer job. + */ + async function checkLatestTransferOperation() { + const filter = JSON.stringify({ + project_id: projectId, + job_names: [jobName], + }); + + const [operations] = await client.operationsClient.listOperations({ + name: 'transferOperations', + filter, + }); + + console.log(`Transfer operations for ${jobName}:`); + for (const {metadata} of operations) { + const transferOperation = TransferOperation.decode(metadata.value); + + console.dir(transferOperation); + } + } + + checkLatestTransferOperation(); + // [END storagetransfer_transfer_check] +} + +main(...process.argv.slice(2)); + +process.on('unhandledRejection', err => { + console.error(err); + process.exitCode = 1; +}); From 928c6e102541834f47e505feff0b946d676bfe1e Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 11 Apr 2022 15:33:00 -0700 Subject: [PATCH 09/27] chore(main): release 1.2.0 (#45) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> Co-authored-by: Daniel Bankhead --- storagetransfer/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storagetransfer/package.json b/storagetransfer/package.json index 470f7a8ba8..2f09ebb4e2 100644 --- a/storagetransfer/package.json +++ b/storagetransfer/package.json @@ -13,7 +13,7 @@ "test": "c8 mocha ---parallel --timeout 600000 test/*.test.js" }, "dependencies": { - "@google-cloud/storage-transfer": "^1.1.1" + "@google-cloud/storage-transfer": "^1.2.0" }, "devDependencies": { "@google-cloud/storage": "^5.15.3", From 9c8f42028f74c7459cb869f59177212a89782b28 Mon Sep 17 00:00:00 2001 From: Daniel Bankhead Date: Mon, 25 Apr 2022 07:57:22 -0700 Subject: [PATCH 10/27] test: Wait for IAM Propagation (#59) --- storagetransfer/test/utils/bucket.js | 56 ++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/storagetransfer/test/utils/bucket.js b/storagetransfer/test/utils/bucket.js index 4a6c26567b..8e2668266f 100644 --- a/storagetransfer/test/utils/bucket.js +++ b/storagetransfer/test/utils/bucket.js @@ -66,17 +66,22 @@ class BucketManager { * Configures permissions for STS to read/write to the bucket. * * @param {Bucket} bucket + * @param {number} waitForPropagation the time in milliseconds, if any, to wait + * for the policy to propigate (default 7 minutes). See for details: + * - https://cloud.google.com/iam/docs/faq#access_revoke + * - https://cloud.google.com/iam/docs/policies#structure */ - async grantSTSPermissions(bucket) { + async grantSTSPermissions(bucket, waitForPropagation = 7 * 60 * 1000) { const [serviceAccount] = await this.client.getGoogleServiceAccount({ projectId: await this.getProjectId(), }); - const email = serviceAccount.accountEmail; + + const member = `serviceAccount:${serviceAccount.accountEmail}`; const objectViewer = 'roles/storage.objectViewer'; const bucketReader = 'roles/storage.legacyBucketReader'; const bucketWriter = 'roles/storage.legacyBucketWriter'; - const members = [`serviceAccount:${email}`]; + const members = [member]; const [policy] = await bucket.iam.getPolicy({requestedPolicyVersion: 3}); @@ -96,6 +101,51 @@ class BucketManager { }); await bucket.iam.setPolicy(policy); + + if (waitForPropagation) { + const limit = Date.now() + waitForPropagation; + + let hasObjectViewer = false; + let hasBucketReader = false; + let hasBucketWriter = false; + + let attempts = 0; + + // Using do/while to ensure this runs at least once + do { + const [policy] = await bucket.iam.getPolicy({ + requestedPolicyVersion: 3, + }); + + for (const item of policy.bindings) { + if (item.members.includes(member)) { + switch (item.role) { + case objectViewer: + hasObjectViewer = true; + break; + case bucketReader: + hasBucketReader = true; + break; + case bucketWriter: + hasBucketWriter = true; + break; + } + } + } + + if (!hasObjectViewer || !hasBucketReader || !hasBucketWriter) { + await Promise(resolve => setTimeout(resolve, 1000 * ++attempts)); + } else { + break; + } + } while (Date.now() < limit); + + if (!hasObjectViewer || !hasBucketReader || !hasBucketWriter) { + throw new RangeError( + `'${member}' is missing the required permissions for bucket '${bucket.name()}'` + ); + } + } } /** From 157d8cf7b7bae3cdfbfb46d1d565b2fdd4733d51 Mon Sep 17 00:00:00 2001 From: sofisl <55454395+sofisl@users.noreply.github.com> Date: Wed, 18 May 2022 17:43:37 -0700 Subject: [PATCH 11/27] build!: update library to use Node 12 (#64) * build!: Update library to use Node 12 Co-authored-by: Owl Bot --- storagetransfer/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storagetransfer/package.json b/storagetransfer/package.json index 2f09ebb4e2..9bce6a4706 100644 --- a/storagetransfer/package.json +++ b/storagetransfer/package.json @@ -4,7 +4,7 @@ "license": "Apache-2.0", "author": "Google LLC", "engines": { - "node": ">=10" + "node": ">=12.0.0" }, "files": [ "*.js" From b09157df23c4aeec8d8610819c3c420743d6ce16 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Tue, 31 May 2022 17:44:36 +0200 Subject: [PATCH 12/27] chore(deps): update dependency @google-cloud/storage to v6 (#66) --- storagetransfer/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storagetransfer/package.json b/storagetransfer/package.json index 9bce6a4706..d732960d5d 100644 --- a/storagetransfer/package.json +++ b/storagetransfer/package.json @@ -16,7 +16,7 @@ "@google-cloud/storage-transfer": "^1.2.0" }, "devDependencies": { - "@google-cloud/storage": "^5.15.3", + "@google-cloud/storage": "^6.0.0", "aws-sdk": "^2.1073.0", "c8": "^7.1.0", "chai": "^4.2.0", From 7d0e1ce949005d018131e5206d68296823ef8ca0 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 31 May 2022 09:25:08 -0700 Subject: [PATCH 13/27] chore(main): release 2.0.0 (#65) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(main): release 2.0.0 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> Co-authored-by: Owl Bot --- storagetransfer/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storagetransfer/package.json b/storagetransfer/package.json index d732960d5d..ff4a002633 100644 --- a/storagetransfer/package.json +++ b/storagetransfer/package.json @@ -13,7 +13,7 @@ "test": "c8 mocha ---parallel --timeout 600000 test/*.test.js" }, "dependencies": { - "@google-cloud/storage-transfer": "^1.2.0" + "@google-cloud/storage-transfer": "^2.0.0" }, "devDependencies": { "@google-cloud/storage": "^6.0.0", From 882d0ce92ec8716d8cb7c4a53d5c68a948af593e Mon Sep 17 00:00:00 2001 From: Daniel Bankhead Date: Tue, 14 Jun 2022 14:53:49 -0700 Subject: [PATCH 14/27] feat(samples): Add POSIX & Manifest samples (#67) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(samples): Add POSIX samples * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * Fix: `fs.promises` import * fix: clean-up created transfer job * chore: Add (safe) debug log * fix: misc bugs * chore: remove debug log * feat: add POSIX to POSIX sample - Also, update tests and comments * chore: typo & clean-up * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * refactor: styling * feat: Add POSIX Download sample * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * fix: Add ending '/' * style: shorten variable * feat: Transfer Manifest request * fix: typo * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * fix: `replaceAll` (not available in Node 12) -> `replace` Co-authored-by: Owl Bot --- storagetransfer/manifest-request.js | 99 ++++++++++++++++ storagetransfer/posix-download.js | 99 ++++++++++++++++ storagetransfer/posix-request.js | 92 +++++++++++++++ storagetransfer/posix-to-posix-request.js | 106 ++++++++++++++++++ storagetransfer/test/manifest-request.test.js | 93 +++++++++++++++ storagetransfer/test/posix-download.test.js | 89 +++++++++++++++ storagetransfer/test/posix-request.test.js | 79 +++++++++++++ .../test/posix-to-posix-request.test.js | 89 +++++++++++++++ 8 files changed, 746 insertions(+) create mode 100644 storagetransfer/manifest-request.js create mode 100644 storagetransfer/posix-download.js create mode 100644 storagetransfer/posix-request.js create mode 100644 storagetransfer/posix-to-posix-request.js create mode 100644 storagetransfer/test/manifest-request.test.js create mode 100644 storagetransfer/test/posix-download.test.js create mode 100644 storagetransfer/test/posix-request.test.js create mode 100644 storagetransfer/test/posix-to-posix-request.test.js diff --git a/storagetransfer/manifest-request.js b/storagetransfer/manifest-request.js new file mode 100644 index 0000000000..3817bfd66a --- /dev/null +++ b/storagetransfer/manifest-request.js @@ -0,0 +1,99 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +async function main( + projectId = 'my-project', + sourceAgentPoolName = '', + rootDirectory = '', + gcsSinkBucket = '', + manifestLocation = '' +) { + // [START storagetransfer_manifest_request] + + // Imports the Google Cloud client library + const { + StorageTransferServiceClient, + } = require('@google-cloud/storage-transfer'); + + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // Your project id + // const projectId = 'my-project' + + // The agent pool associated with the POSIX data source. Defaults to the default agent + // const sourceAgentPoolName = 'projects/my-project/agentPools/transfer_service_default' + + // The root directory path on the source filesystem + // const rootDirectory = '/directory/to/transfer/source', + + // The ID of the GCS bucket to transfer data to + // const gcsSinkBucket = 'my-sink-bucket' + + // Transfer manifest location. Must be a + // const manifestLocation = 'gs://my-bucket/sample_manifest.csv' + + // Creates a client + const client = new StorageTransferServiceClient(); + + /** + * Creates a request to transfer from the local file system to the sink bucket + */ + async function transferDirectory() { + const createRequest = { + transferJob: { + projectId, + transferSpec: { + sourceAgentPoolName, + posixDataSource: { + rootDirectory, + }, + gcsDataSink: {bucketName: gcsSinkBucket}, + transferManifest: { + location: manifestLocation, + }, + }, + status: 'ENABLED', + }, + }; + + // Runs the request and creates the job + const [transferJob] = await client.createTransferJob(createRequest); + + const runRequest = { + jobName: transferJob.name, + projectId: projectId, + }; + + await client.runTransferJob(runRequest); + + console.log( + `Created and ran a transfer job from '${rootDirectory}' to '${gcsSinkBucket}' using manifest \`${manifestLocation}\` with name ${transferJob.name}` + ); + } + + transferDirectory(); + // [END storagetransfer_manifest_request] +} + +main(...process.argv.slice(2)); + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); diff --git a/storagetransfer/posix-download.js b/storagetransfer/posix-download.js new file mode 100644 index 0000000000..818de46934 --- /dev/null +++ b/storagetransfer/posix-download.js @@ -0,0 +1,99 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +async function main( + projectId = 'my-project', + sinkAgentPoolName = '', + gcsSourceBucket = '', + gcsSourcePath = '', + rootDirectory = '' +) { + // [START storagetransfer_download_to_posix] + + // Imports the Google Cloud client library + const { + StorageTransferServiceClient, + } = require('@google-cloud/storage-transfer'); + + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // Your project id + // const projectId = 'my-project' + + // The agent pool associated with the POSIX data sink. Defaults to the default agent + // const sinkAgentPoolName = 'projects/my-project/agentPools/transfer_service_default' + + // Google Cloud Storage source bucket name + // const gcsSourceBucket = 'my-gcs-source-bucket' + + // An optional path on the Google Cloud Storage bucket to download from + // const gcsSourcePath = 'foo/bar/' + + // The root directory path on the source filesystem + // const rootDirectory = '/directory/to/transfer/source', + + // Creates a client + const client = new StorageTransferServiceClient(); + + /** + * Creates a request to transfer from the local file system to the sink bucket + */ + async function transferDirectory() { + const createRequest = { + transferJob: { + projectId, + transferSpec: { + sinkAgentPoolName, + gcsDataSource: { + bucketName: gcsSourceBucket, + path: gcsSourcePath, + }, + posixDataSink: { + rootDirectory, + }, + }, + status: 'ENABLED', + }, + }; + + // Runs the request and creates the job + const [transferJob] = await client.createTransferJob(createRequest); + + const runRequest = { + jobName: transferJob.name, + projectId: projectId, + }; + + await client.runTransferJob(runRequest); + + console.log( + `Downloading from '${gcsSourceBucket}' (path: \`${gcsSourcePath}\`) to '${rootDirectory}' with name ${transferJob.name}` + ); + } + + transferDirectory(); + // [END storagetransfer_download_to_posix] +} + +main(...process.argv.slice(2)); + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); diff --git a/storagetransfer/posix-request.js b/storagetransfer/posix-request.js new file mode 100644 index 0000000000..8ff8ee6fce --- /dev/null +++ b/storagetransfer/posix-request.js @@ -0,0 +1,92 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +async function main( + projectId = 'my-project', + sourceAgentPoolName = '', + rootDirectory = '', + gcsSinkBucket = '' +) { + // [START storagetransfer_transfer_from_posix] + + // Imports the Google Cloud client library + const { + StorageTransferServiceClient, + } = require('@google-cloud/storage-transfer'); + + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // Your project id + // const projectId = 'my-project' + + // The agent pool associated with the POSIX data source. Defaults to the default agent + // const sourceAgentPoolName = 'projects/my-project/agentPools/transfer_service_default' + + // The root directory path on the source filesystem + // const rootDirectory = '/directory/to/transfer/source', + + // The ID of the GCS bucket to transfer data to + // const gcsSinkBucket = 'my-sink-bucket' + + // Creates a client + const client = new StorageTransferServiceClient(); + + /** + * Creates a request to transfer from the local file system to the sink bucket + */ + async function transferDirectory() { + const createRequest = { + transferJob: { + projectId, + transferSpec: { + sourceAgentPoolName, + posixDataSource: { + rootDirectory, + }, + gcsDataSink: {bucketName: gcsSinkBucket}, + }, + status: 'ENABLED', + }, + }; + + // Runs the request and creates the job + const [transferJob] = await client.createTransferJob(createRequest); + + const runRequest = { + jobName: transferJob.name, + projectId: projectId, + }; + + await client.runTransferJob(runRequest); + + console.log( + `Created and ran a transfer job from '${rootDirectory}' to '${gcsSinkBucket}' with name ${transferJob.name}` + ); + } + + transferDirectory(); + // [END storagetransfer_transfer_from_posix] +} + +main(...process.argv.slice(2)); + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); diff --git a/storagetransfer/posix-to-posix-request.js b/storagetransfer/posix-to-posix-request.js new file mode 100644 index 0000000000..50f3097f34 --- /dev/null +++ b/storagetransfer/posix-to-posix-request.js @@ -0,0 +1,106 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +async function main( + projectId = 'my-project', + sourceAgentPoolName = '', + sinkAgentPoolName = '', + rootDirectory = '', + destinationDirectory = '', + bucketName = '' +) { + // [START storagetransfer_transfer_posix_to_posix] + + // Imports the Google Cloud client library + const { + StorageTransferServiceClient, + } = require('@google-cloud/storage-transfer'); + + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // Your project id + // const projectId = 'my-project' + + // The agent pool associated with the POSIX data source. Defaults to the default agent + // const sourceAgentPoolName = 'projects/my-project/agentPools/transfer_service_default' + + // The agent pool associated with the POSIX data sink. Defaults to the default agent + // const sinkAgentPoolName = 'projects/my-project/agentPools/transfer_service_default' + + // The root directory path on the source filesystem + // const rootDirectory = '/directory/to/transfer/source', + + // The root directory path on the sink filesystem + // const destinationDirectory = '/directory/to/transfer/sink' + + // The ID of the GCS bucket for intermediate storage + // const bucketName = 'my-intermediate-bucket' + + // Creates a client + const client = new StorageTransferServiceClient(); + + /** + * Creates a request to transfer from the local file system to the sink bucket + */ + async function transferDirectory() { + const createRequest = { + transferJob: { + projectId, + transferSpec: { + sourceAgentPoolName, + sinkAgentPoolName, + posixDataSource: { + rootDirectory, + }, + posixDataSink: { + rootDirectory: destinationDirectory, + }, + gcsIntermediateDataLocation: { + bucketName, + }, + }, + status: 'ENABLED', + }, + }; + + // Runs the request and creates the job + const [transferJob] = await client.createTransferJob(createRequest); + + const runRequest = { + jobName: transferJob.name, + projectId: projectId, + }; + + await client.runTransferJob(runRequest); + + console.log( + `Created and ran a transfer job from '${rootDirectory}' to '${destinationDirectory}' with name ${transferJob.name}` + ); + } + + transferDirectory(); + // [END storagetransfer_transfer_posix_to_posix] +} + +main(...process.argv.slice(2)); + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); diff --git a/storagetransfer/test/manifest-request.test.js b/storagetransfer/test/manifest-request.test.js new file mode 100644 index 0000000000..b0db4a4219 --- /dev/null +++ b/storagetransfer/test/manifest-request.test.js @@ -0,0 +1,93 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const fs = require('fs').promises; +const os = require('os'); +const path = require('path'); + +const {assert} = require('chai'); +const {after, before, describe, it} = require('mocha'); + +const {BucketManager, TransferJobManager, runSample} = require('./utils'); + +describe('manifest-request', () => { + const testBucketManager = new BucketManager(); + const testTransferJobManager = new TransferJobManager(); + + let projectId; + let sourceAgentPoolName; + let rootDirectory; + let gcsSinkBucket; + let manifestLocation; + + let tempManifestObject; + let tempFile; + + before(async () => { + projectId = await testTransferJobManager.client.getProjectId(); + + // Use default pool + sourceAgentPoolName = ''; + + rootDirectory = await fs.mkdtemp( + path.join(os.tmpdir(), 'sts-manifest-request-test-src-') + ); + + const bucket = await testBucketManager.generateGCSBucket(); + + gcsSinkBucket = bucket.name; + + tempFile = path.join(rootDirectory, 'text.txt'); + await fs.writeFile(tempFile, 'test data'); + + // Double-quote to escape double-quotes in CSV text + const csvContent = `"${tempFile.replace(/"/g, '""')}"`; + + tempManifestObject = bucket.file('manifest.csv'); + await tempManifestObject.save(csvContent); + + manifestLocation = `gs://${bucket.name}/${tempManifestObject.name}`; + }); + + after(async () => { + await tempManifestObject.delete(); + await testBucketManager.deleteBuckets(); + await testTransferJobManager.cleanUp(); + await fs.unlink(tempFile); + await fs.rmdir(rootDirectory); + }); + + it('should create a transfer job with a manifest from POSIX to GCS', async () => { + const output = await runSample('manifest-request', [ + projectId, + sourceAgentPoolName, + rootDirectory, + gcsSinkBucket, + manifestLocation, + ]); + + // If it ran successfully and a job was created, delete it to clean up + const [jobName] = output.match(/transferJobs.*/); + if (jobName) { + testTransferJobManager.transferJobToCleanUp(jobName); + } + + // Find at least 1 transfer operation from the transfer job in the output + assert.include(output, 'Created and ran a transfer job'); + }); +}); diff --git a/storagetransfer/test/posix-download.test.js b/storagetransfer/test/posix-download.test.js new file mode 100644 index 0000000000..bfe904f20b --- /dev/null +++ b/storagetransfer/test/posix-download.test.js @@ -0,0 +1,89 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const fs = require('fs').promises; +const os = require('os'); +const path = require('path'); + +const {assert} = require('chai'); +const {after, before, describe, it} = require('mocha'); + +const {BucketManager, TransferJobManager, runSample} = require('./utils'); + +describe('posix-download', () => { + const testBucketManager = new BucketManager(); + const testTransferJobManager = new TransferJobManager(); + + let projectId; + let sinkAgentPoolName; + let gcsSourceBucket; + let gcsSourcePath; + let rootDirectory; + + let tempObject; + + before(async () => { + projectId = await testTransferJobManager.client.getProjectId(); + + // Use default pool + sinkAgentPoolName = ''; + + const bucket = await testBucketManager.generateGCSBucket(); + gcsSourceBucket = bucket.name; + + rootDirectory = await fs.mkdtemp( + path.join(os.tmpdir(), 'sts-posix-download-test-sink-') + ); + + // API requires path to end with '/' + gcsSourcePath = rootDirectory + path.posix.sep; + + tempObject = bucket.file(path.join(rootDirectory, 'text.txt')); + + await tempObject.save('test data'); + }); + + after(async () => { + await tempObject.delete(); + await testBucketManager.deleteBuckets(); + await testTransferJobManager.cleanUp(); + await fs.rmdir(rootDirectory); + }); + + it('should create a transfer job from GCS to POSIX', async () => { + const output = await runSample('posix-download', [ + projectId, + sinkAgentPoolName, + gcsSourceBucket, + gcsSourcePath, + rootDirectory, + ]); + + // If it ran successfully and a job was created, delete it to clean up + const [jobName] = output.match(/transferJobs.*/); + if (jobName) { + testTransferJobManager.transferJobToCleanUp(jobName); + } + + // Find at least 1 transfer operation from the transfer job in the output + assert.include( + output, + `Downloading from '${gcsSourceBucket}' (path: \`${gcsSourcePath}\`) to '${rootDirectory}'` + ); + }); +}); diff --git a/storagetransfer/test/posix-request.test.js b/storagetransfer/test/posix-request.test.js new file mode 100644 index 0000000000..d7e0d1181d --- /dev/null +++ b/storagetransfer/test/posix-request.test.js @@ -0,0 +1,79 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const fs = require('fs').promises; +const os = require('os'); +const path = require('path'); + +const {assert} = require('chai'); +const {after, before, describe, it} = require('mocha'); + +const {BucketManager, TransferJobManager, runSample} = require('./utils'); + +describe('posix-request', () => { + const testBucketManager = new BucketManager(); + const testTransferJobManager = new TransferJobManager(); + + let projectId; + let sourceAgentPoolName; + let rootDirectory; + let gcsSinkBucket; + + let tempFile; + + before(async () => { + projectId = await testTransferJobManager.client.getProjectId(); + + // Use default pool + sourceAgentPoolName = ''; + + rootDirectory = await fs.mkdtemp( + path.join(os.tmpdir(), 'sts-posix-request-test-src-') + ); + + gcsSinkBucket = (await testBucketManager.generateGCSBucket()).name; + + tempFile = path.join(rootDirectory, 'text.txt'); + await fs.writeFile(tempFile, 'test data'); + }); + + after(async () => { + await testBucketManager.deleteBuckets(); + await testTransferJobManager.cleanUp(); + await fs.unlink(tempFile); + await fs.rmdir(rootDirectory); + }); + + it('should create a transfer job from POSIX to GCS', async () => { + const output = await runSample('posix-request', [ + projectId, + sourceAgentPoolName, + rootDirectory, + gcsSinkBucket, + ]); + + // If it ran successfully and a job was created, delete it to clean up + const [jobName] = output.match(/transferJobs.*/); + if (jobName) { + testTransferJobManager.transferJobToCleanUp(jobName); + } + + // Find at least 1 transfer operation from the transfer job in the output + assert.include(output, 'Created and ran a transfer job'); + }); +}); diff --git a/storagetransfer/test/posix-to-posix-request.test.js b/storagetransfer/test/posix-to-posix-request.test.js new file mode 100644 index 0000000000..da7fd9ad69 --- /dev/null +++ b/storagetransfer/test/posix-to-posix-request.test.js @@ -0,0 +1,89 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const fs = require('fs').promises; +const os = require('os'); +const path = require('path'); + +const {assert} = require('chai'); +const {after, before, describe, it} = require('mocha'); + +const {BucketManager, TransferJobManager, runSample} = require('./utils'); + +describe('posix-to-posix-request', () => { + const testBucketManager = new BucketManager(); + const testTransferJobManager = new TransferJobManager(); + + let projectId; + let sourceAgentPoolName; + let sinkAgentPoolName; + let rootDirectory; + let destinationDirectory; + let bucketName; + + let tempFile; + + before(async () => { + projectId = await testTransferJobManager.client.getProjectId(); + + // Use default pool + sourceAgentPoolName = ''; + sinkAgentPoolName = ''; + + rootDirectory = await fs.mkdtemp( + path.join(os.tmpdir(), 'sts-posix-to-posix-request-test-src-') + ); + + destinationDirectory = await fs.mkdtemp( + path.join(os.tmpdir(), 'sts-posix-to-posix-request-test-sink-') + ); + + bucketName = (await testBucketManager.generateGCSBucket()).name; + + tempFile = path.join(rootDirectory, 'text.txt'); + await fs.writeFile(tempFile, 'test data'); + }); + + after(async () => { + await testBucketManager.deleteBuckets(); + await testTransferJobManager.cleanUp(); + + await fs.unlink(tempFile); + await fs.rmdir(rootDirectory); + }); + + it('should create a transfer job from POSIX to POSIX', async () => { + const output = await runSample('posix-to-posix-request', [ + projectId, + sourceAgentPoolName, + sinkAgentPoolName, + rootDirectory, + destinationDirectory, + bucketName, + ]); + + // If it ran successfully and a job was created, delete it to clean up + const [jobName] = output.match(/transferJobs.*/); + if (jobName) { + testTransferJobManager.transferJobToCleanUp(jobName); + } + + // Find at least 1 transfer operation from the transfer job in the output + assert.include(output, 'Created and ran a transfer job'); + }); +}); From 2e76e91ccacd2dae1da2f65c76f12d48f2613fc9 Mon Sep 17 00:00:00 2001 From: Daniel Bankhead Date: Thu, 16 Jun 2022 11:17:10 -0700 Subject: [PATCH 15/27] refactor(samples): Fix documentation, typos, and function names (#72) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(samples): Add POSIX samples * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * Fix: `fs.promises` import * fix: clean-up created transfer job * chore: Add (safe) debug log * fix: misc bugs * chore: remove debug log * feat: add POSIX to POSIX sample - Also, update tests and comments * chore: typo & clean-up * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * refactor: styling * feat: Add POSIX Download sample * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * fix: Add ending '/' * style: shorten variable * feat: Transfer Manifest request * fix: typo * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * fix: `replaceAll` (not available in Node 12) -> `replace` * refactor(samples): Fix documentation, typos, and function names * fix: typo Co-authored-by: Owl Bot --- storagetransfer/manifest-request.js | 8 ++++---- storagetransfer/posix-download.js | 8 ++++---- storagetransfer/posix-request.js | 2 +- storagetransfer/posix-to-posix-request.js | 2 +- storagetransfer/test/manifest-request.test.js | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/storagetransfer/manifest-request.js b/storagetransfer/manifest-request.js index 3817bfd66a..a76909c60d 100644 --- a/storagetransfer/manifest-request.js +++ b/storagetransfer/manifest-request.js @@ -40,12 +40,12 @@ async function main( // const sourceAgentPoolName = 'projects/my-project/agentPools/transfer_service_default' // The root directory path on the source filesystem - // const rootDirectory = '/directory/to/transfer/source', + // const rootDirectory = '/directory/to/transfer/source' // The ID of the GCS bucket to transfer data to // const gcsSinkBucket = 'my-sink-bucket' - // Transfer manifest location. Must be a + // Transfer manifest location. Must be a `gs:` URL // const manifestLocation = 'gs://my-bucket/sample_manifest.csv' // Creates a client @@ -54,7 +54,7 @@ async function main( /** * Creates a request to transfer from the local file system to the sink bucket */ - async function transferDirectory() { + async function transferViaManifest() { const createRequest = { transferJob: { projectId, @@ -87,7 +87,7 @@ async function main( ); } - transferDirectory(); + transferViaManifest(); // [END storagetransfer_manifest_request] } diff --git a/storagetransfer/posix-download.js b/storagetransfer/posix-download.js index 818de46934..3314992288 100644 --- a/storagetransfer/posix-download.js +++ b/storagetransfer/posix-download.js @@ -45,8 +45,8 @@ async function main( // An optional path on the Google Cloud Storage bucket to download from // const gcsSourcePath = 'foo/bar/' - // The root directory path on the source filesystem - // const rootDirectory = '/directory/to/transfer/source', + // The root directory path on the destination filesystem + // const rootDirectory = '/directory/to/transfer/sink' // Creates a client const client = new StorageTransferServiceClient(); @@ -54,7 +54,7 @@ async function main( /** * Creates a request to transfer from the local file system to the sink bucket */ - async function transferDirectory() { + async function downloadFromGCS() { const createRequest = { transferJob: { projectId, @@ -87,7 +87,7 @@ async function main( ); } - transferDirectory(); + downloadFromGCS(); // [END storagetransfer_download_to_posix] } diff --git a/storagetransfer/posix-request.js b/storagetransfer/posix-request.js index 8ff8ee6fce..e405c6f8c9 100644 --- a/storagetransfer/posix-request.js +++ b/storagetransfer/posix-request.js @@ -39,7 +39,7 @@ async function main( // const sourceAgentPoolName = 'projects/my-project/agentPools/transfer_service_default' // The root directory path on the source filesystem - // const rootDirectory = '/directory/to/transfer/source', + // const rootDirectory = '/directory/to/transfer/source' // The ID of the GCS bucket to transfer data to // const gcsSinkBucket = 'my-sink-bucket' diff --git a/storagetransfer/posix-to-posix-request.js b/storagetransfer/posix-to-posix-request.js index 50f3097f34..8228f20998 100644 --- a/storagetransfer/posix-to-posix-request.js +++ b/storagetransfer/posix-to-posix-request.js @@ -44,7 +44,7 @@ async function main( // const sinkAgentPoolName = 'projects/my-project/agentPools/transfer_service_default' // The root directory path on the source filesystem - // const rootDirectory = '/directory/to/transfer/source', + // const rootDirectory = '/directory/to/transfer/source' // The root directory path on the sink filesystem // const destinationDirectory = '/directory/to/transfer/sink' diff --git a/storagetransfer/test/manifest-request.test.js b/storagetransfer/test/manifest-request.test.js index b0db4a4219..8f147fa362 100644 --- a/storagetransfer/test/manifest-request.test.js +++ b/storagetransfer/test/manifest-request.test.js @@ -72,7 +72,7 @@ describe('manifest-request', () => { await fs.rmdir(rootDirectory); }); - it('should create a transfer job with a manifest from POSIX to GCS', async () => { + it('should create a transfer job using a manifest from POSIX to GCS', async () => { const output = await runSample('manifest-request', [ projectId, sourceAgentPoolName, From fc4dc12ba518b8fceed5ba29f8f3c9644179272c Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 30 Jun 2022 19:18:32 +0000 Subject: [PATCH 16/27] chore(main): release 2.1.0 (#76) :robot: I have created a release *beep* *boop* --- ## [2.1.0](https://github.com/googleapis/nodejs-storage-transfer/compare/v2.0.0...v2.1.0) (2022-06-30) ### Features * **samples:** Add POSIX & Manifest samples ([#67](https://github.com/googleapis/nodejs-storage-transfer/issues/67)) ([1056c9f](https://github.com/googleapis/nodejs-storage-transfer/commit/1056c9fde90c0c7b79a235d4c9f5f861efce3169)) * support regapic LRO ([#75](https://github.com/googleapis/nodejs-storage-transfer/issues/75)) ([cc80906](https://github.com/googleapis/nodejs-storage-transfer/commit/cc80906bcf86b68485fd62970b1b6f8a30c1123b)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- storagetransfer/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storagetransfer/package.json b/storagetransfer/package.json index ff4a002633..f8960f3487 100644 --- a/storagetransfer/package.json +++ b/storagetransfer/package.json @@ -13,7 +13,7 @@ "test": "c8 mocha ---parallel --timeout 600000 test/*.test.js" }, "dependencies": { - "@google-cloud/storage-transfer": "^2.0.0" + "@google-cloud/storage-transfer": "^2.1.0" }, "devDependencies": { "@google-cloud/storage": "^6.0.0", From e712a415d4b6e1e45fa97145681ade9412808aca Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Fri, 26 Aug 2022 12:24:45 -0700 Subject: [PATCH 17/27] chore(main): release 2.1.1 (#86) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(main): release 2.1.1 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> Co-authored-by: Owl Bot --- storagetransfer/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storagetransfer/package.json b/storagetransfer/package.json index f8960f3487..0acd2a6296 100644 --- a/storagetransfer/package.json +++ b/storagetransfer/package.json @@ -13,7 +13,7 @@ "test": "c8 mocha ---parallel --timeout 600000 test/*.test.js" }, "dependencies": { - "@google-cloud/storage-transfer": "^2.1.0" + "@google-cloud/storage-transfer": "^2.1.1" }, "devDependencies": { "@google-cloud/storage": "^6.0.0", From 0514f36fdde86dcd62ed6eb2554a1ac1034a0af7 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 7 Sep 2022 18:06:31 -0400 Subject: [PATCH 18/27] chore(main): release 2.1.2 (#91) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(main): release 2.1.2 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> Co-authored-by: Owl Bot --- storagetransfer/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storagetransfer/package.json b/storagetransfer/package.json index 0acd2a6296..f00f2c0316 100644 --- a/storagetransfer/package.json +++ b/storagetransfer/package.json @@ -13,7 +13,7 @@ "test": "c8 mocha ---parallel --timeout 600000 test/*.test.js" }, "dependencies": { - "@google-cloud/storage-transfer": "^2.1.1" + "@google-cloud/storage-transfer": "^2.1.2" }, "devDependencies": { "@google-cloud/storage": "^6.0.0", From fc6352e239285d710f0736c4215e4bfd74af1954 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 9 Sep 2022 04:04:31 +0200 Subject: [PATCH 19/27] chore(deps): update dependency uuid to v9 (#92) --- storagetransfer/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storagetransfer/package.json b/storagetransfer/package.json index f00f2c0316..770a78fff6 100644 --- a/storagetransfer/package.json +++ b/storagetransfer/package.json @@ -21,6 +21,6 @@ "c8": "^7.1.0", "chai": "^4.2.0", "mocha": "^9.0.0", - "uuid": "^8.0.0" + "uuid": "^9.0.0" } } From 19a699fdbde7f60f57708b2dbb86447491e01724 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 3 Oct 2022 15:07:02 -0700 Subject: [PATCH 20/27] chore(main): release 2.2.0 (#95) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(main): release 2.2.0 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> Co-authored-by: Owl Bot --- storagetransfer/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storagetransfer/package.json b/storagetransfer/package.json index 770a78fff6..3e6cf13634 100644 --- a/storagetransfer/package.json +++ b/storagetransfer/package.json @@ -13,7 +13,7 @@ "test": "c8 mocha ---parallel --timeout 600000 test/*.test.js" }, "dependencies": { - "@google-cloud/storage-transfer": "^2.1.2" + "@google-cloud/storage-transfer": "^2.2.0" }, "devDependencies": { "@google-cloud/storage": "^6.0.0", From 09a575aca70d811844275d6ba07a7b8dd9964b69 Mon Sep 17 00:00:00 2001 From: Daniel Bankhead Date: Thu, 6 Oct 2022 12:34:52 -0700 Subject: [PATCH 21/27] refactor(samples): pass AWS credentials via environment variables (#116) --- storagetransfer/aws-request.js | 6 +++--- storagetransfer/test/aws-request.test.js | 9 +-------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/storagetransfer/aws-request.js b/storagetransfer/aws-request.js index 296e120e99..0cfc9121f9 100644 --- a/storagetransfer/aws-request.js +++ b/storagetransfer/aws-request.js @@ -20,9 +20,9 @@ async function main( projectId, description, awsSourceBucket, - awsAccessKeyId, - awsSecretAccessKey, - gcsSinkBucket + gcsSinkBucket, + awsAccessKeyId = process.env.AWS_ACCESS_KEY_ID, + awsSecretAccessKey = process.env.AWS_SECRET_ACCESS_KEY ) { // [START storagetransfer_transfer_from_aws] diff --git a/storagetransfer/test/aws-request.test.js b/storagetransfer/test/aws-request.test.js index ce5e37e720..3e68d6e151 100644 --- a/storagetransfer/test/aws-request.test.js +++ b/storagetransfer/test/aws-request.test.js @@ -28,16 +28,11 @@ describe('aws-request', () => { let projectId; let description; let awsSourceBucket; - let awsAccessKeyId; - let awsSecretAccessKey; let gcsSinkBucket; before(async () => { - awsAccessKeyId = process.env.AWS_ACCESS_KEY_ID; - awsSecretAccessKey = process.env.AWS_SECRET_ACCESS_KEY; - assert( - awsAccessKeyId && awsSecretAccessKey, + process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY, 'environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are required' ); @@ -57,8 +52,6 @@ describe('aws-request', () => { projectId, description, awsSourceBucket, - awsAccessKeyId, - awsSecretAccessKey, gcsSinkBucket, ]); From ace1413a0d85f9da695f2d7671ea6c23461c3b68 Mon Sep 17 00:00:00 2001 From: Daniel Bankhead Date: Wed, 12 Oct 2022 14:56:04 -0700 Subject: [PATCH 22/27] samples: Add AWS S3-compatible Sample (#118) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * samples: Add AWS S3-compatible sample * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * docs: description * docs: typo * docs: another typo * docs: typos Co-authored-by: Owl Bot --- .../aws-s3-compatible-source-request.js | 132 ++++++++++++++++++ .../aws-s3-compatible-source-request.test.js | 91 ++++++++++++ storagetransfer/test/utils/transfer.js | 8 ++ 3 files changed, 231 insertions(+) create mode 100644 storagetransfer/aws-s3-compatible-source-request.js create mode 100644 storagetransfer/test/aws-s3-compatible-source-request.test.js diff --git a/storagetransfer/aws-s3-compatible-source-request.js b/storagetransfer/aws-s3-compatible-source-request.js new file mode 100644 index 0000000000..460a32a4ee --- /dev/null +++ b/storagetransfer/aws-s3-compatible-source-request.js @@ -0,0 +1,132 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const {protos} = require('@google-cloud/storage-transfer'); +const {AuthMethod, NetworkProtocol, RequestModel} = + protos.google.storagetransfer.v1.S3CompatibleMetadata; + +async function main( + projectId = 'my-project', + sourceAgentPoolName = 'projects/my-project/agentPools/transfer_service_default', + sourceBucketName = 'my-bucket-name', + sourcePath = 'path/to/data/', + gcsSinkBucket = 'my-sink-bucket', + gcsPath = 'path/to/data/', + region = 'us-east-1', + endpoint = 'us-east-1.example.com', + protocol = NetworkProtocol.NETWORK_PROTOCOL_HTTPS, + requestModel = RequestModel.REQUEST_MODEL_VIRTUAL_HOSTED_STYLE, + authMethod = AuthMethod.AUTH_METHOD_AWS_SIGNATURE_V4 +) { + // [START storagetransfer_transfer_from_s3_compatible_source] + + // Imports the Google Cloud client library + const storageTransfer = require('@google-cloud/storage-transfer'); + + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // Useful enums for AWS S3-Compatible Transfers + // const {AuthMethod, NetworkProtocol, RequestModel} = storageTransfer.protos.google.storagetransfer.v1.S3CompatibleMetadata; + + // Your project id + // const projectId = 'my-project'; + + // The agent pool associated with the S3-compatible data source. Defaults to the default agent + // const sourceAgentPoolName = 'projects/my-project/agentPools/transfer_service_default'; + + // The S3-compatible bucket name to transfer data from + // const sourceBucketName = "my-bucket-name"; + + // The S3-compatible path (object prefix) to transfer data from + // const sourcePath = "path/to/data/"; + + // The ID of the GCS bucket to transfer data to + // const gcsSinkBucket = "my-sink-bucket"; + + // The GCS path (object prefix) to transfer data to + // const gcsPath = "path/to/data/"; + + // The S3 region of the source bucket + // const region = 'us-east-1'; + + // The S3-compatible endpoint + // const endpoint = "us-east-1.example.com"; + + // The S3-compatible network protocol + // const protocol = NetworkProtocol.NETWORK_PROTOCOL_HTTPS; + + // The S3-compatible request model + // const requestModel = RequestModel.REQUEST_MODEL_VIRTUAL_HOSTED_STYLE; + + // The S3-compatible auth method + // const authMethod = AuthMethod.AUTH_METHOD_AWS_SIGNATURE_V4; + + // Creates a client + const client = new storageTransfer.StorageTransferServiceClient(); + + /** + * Creates a transfer from an AWS S3-compatible source to GCS + */ + async function transferFromS3CompatibleSource() { + // Runs the request and creates the job + const [transferJob] = await client.createTransferJob({ + transferJob: { + projectId, + transferSpec: { + sourceAgentPoolName, + awsS3CompatibleDataSource: { + region, + s3Metadata: { + authMethod, + protocol, + requestModel, + }, + endpoint, + bucketName: sourceBucketName, + path: sourcePath, + }, + gcsDataSink: { + bucketName: gcsSinkBucket, + path: gcsPath, + }, + }, + status: 'ENABLED', + }, + }); + + await client.runTransferJob({ + jobName: transferJob.name, + projectId, + }); + + console.log( + `Created and ran a transfer job from '${sourceBucketName}' to '${gcsSinkBucket}' with name ${transferJob.name}` + ); + } + + transferFromS3CompatibleSource(); + // [END storagetransfer_transfer_from_s3_compatible_source] +} + +main(...process.argv.slice(2)); + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); diff --git a/storagetransfer/test/aws-s3-compatible-source-request.test.js b/storagetransfer/test/aws-s3-compatible-source-request.test.js new file mode 100644 index 0000000000..25a2fd3f0c --- /dev/null +++ b/storagetransfer/test/aws-s3-compatible-source-request.test.js @@ -0,0 +1,91 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const {assert} = require('chai'); +const {after, before, describe, it} = require('mocha'); + +const {BucketManager, TransferJobManager, runSample} = require('./utils'); + +describe('aws-s3-compatible-source-request', () => { + const testBucketManager = new BucketManager(); + const testTransferJobManager = new TransferJobManager(); + const {NetworkProtocol, RequestModel, AuthMethod} = + TransferJobManager.protos.storagetransfer.v1.S3CompatibleMetadata; + + let projectId; + let sourceAgentPoolName; + let sourceBucketName; + let sourcePath; + let gcsSinkBucket; + let gcsPath; + let region; + let endpoint; + let protocol; + let requestModel; + let authMethod; + + before(async () => { + projectId = await testTransferJobManager.client.getProjectId(); + + // Use default pool + sourceAgentPoolName = ''; + + const sourceBucket = await testBucketManager.generateGCSBucket(); + sourceBucketName = sourceBucket.name; + sourcePath = 'path/to/data/'; + + gcsSinkBucket = (await testBucketManager.generateGCSBucket()).name; + gcsPath = 'path/to/data/'; + + region = sourceBucket.getMetadata().location; + endpoint = sourceBucket.baseUrl; + protocol = NetworkProtocol.NETWORK_PROTOCOL_HTTPS; + requestModel = RequestModel.REQUEST_MODEL_VIRTUAL_HOSTED_STYLE; + authMethod = AuthMethod.AUTH_METHOD_AWS_SIGNATURE_V4; + }); + + after(async () => { + await testBucketManager.deleteBuckets(); + await testTransferJobManager.cleanUp(); + }); + + it('should create a transfer job from an AWS S3-compatible source to GCS', async () => { + const output = await runSample('aws-s3-compatible-source-request', [ + projectId, + sourceAgentPoolName, + sourceBucketName, + sourcePath, + gcsSinkBucket, + gcsPath, + region, + endpoint, + protocol, + requestModel, + authMethod, + ]); + + // If it ran successfully and a job was created, delete it to clean up + const [jobName] = output.match(/transferJobs.*/); + if (jobName) { + testTransferJobManager.transferJobToCleanUp(jobName); + } + + // Find at least 1 transfer operation from the transfer job in the output + assert.include(output, 'Created and ran a transfer job'); + }); +}); diff --git a/storagetransfer/test/utils/transfer.js b/storagetransfer/test/utils/transfer.js index f7c1a767ec..80e11fb451 100644 --- a/storagetransfer/test/utils/transfer.js +++ b/storagetransfer/test/utils/transfer.js @@ -18,8 +18,12 @@ const { StorageTransferServiceClient, + protos, } = require('@google-cloud/storage-transfer'); +this.protos = protos.google; +require('@google-cloud/storage-transfer'); + const {BucketManager} = require('./bucket'); class TransferJobManager { @@ -105,6 +109,10 @@ class TransferJobManager { transferJobToCleanUp(jobName) { this.transferJobsToCleanup.push(jobName); } + + static get protos() { + return protos.google; + } } module.exports = {TransferJobManager}; From 122bdbf837a017c10d0d9580014c6dfd541cf1cd Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Fri, 11 Nov 2022 02:46:13 +0000 Subject: [PATCH 23/27] chore(main): release 2.2.1 (#123) :robot: I have created a release *beep* *boop* --- ## [2.2.1](https://togithub.com/googleapis/nodejs-storage-transfer/compare/v2.2.0...v2.2.1) (2022-11-10) ### Bug Fixes * **deps:** Use google-gax v3.5.2 ([#121](https://togithub.com/googleapis/nodejs-storage-transfer/issues/121)) ([ee66be0](https://togithub.com/googleapis/nodejs-storage-transfer/commit/ee66be0297f1665298912c641fa9f2e1b3c2a1a7)) * Regenerated protos JS and TS definitions ([#125](https://togithub.com/googleapis/nodejs-storage-transfer/issues/125)) ([21ad9a7](https://togithub.com/googleapis/nodejs-storage-transfer/commit/21ad9a7190881ecdb8cf68fabcdc4fb42ffc626b)) --- This PR was generated with [Release Please](https://togithub.com/googleapis/release-please). See [documentation](https://togithub.com/googleapis/release-please#release-please). --- storagetransfer/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storagetransfer/package.json b/storagetransfer/package.json index 3e6cf13634..47241a24b8 100644 --- a/storagetransfer/package.json +++ b/storagetransfer/package.json @@ -13,7 +13,7 @@ "test": "c8 mocha ---parallel --timeout 600000 test/*.test.js" }, "dependencies": { - "@google-cloud/storage-transfer": "^2.2.0" + "@google-cloud/storage-transfer": "^2.2.1" }, "devDependencies": { "@google-cloud/storage": "^6.0.0", From 0edfbc4496409f6d4fba71a0198c5a6741d2ca8d Mon Sep 17 00:00:00 2001 From: Daniel Bankhead Date: Mon, 28 Nov 2022 15:28:50 -0800 Subject: [PATCH 24/27] test(samples): Refactor AWS S3 Test Credentials Source (#133) Also allows the S3 client to be configured in a separate step. --- storagetransfer/test/aws-request.test.js | 5 +---- storagetransfer/test/utils/bucket.js | 5 ++++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/storagetransfer/test/aws-request.test.js b/storagetransfer/test/aws-request.test.js index 3e68d6e151..745b2b1af6 100644 --- a/storagetransfer/test/aws-request.test.js +++ b/storagetransfer/test/aws-request.test.js @@ -31,10 +31,7 @@ describe('aws-request', () => { let gcsSinkBucket; before(async () => { - assert( - process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY, - 'environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are required' - ); + testBucketManager.setupS3(); projectId = await testBucketManager.getProjectId(); awsSourceBucket = await testBucketManager.generateS3Bucket(); diff --git a/storagetransfer/test/utils/bucket.js b/storagetransfer/test/utils/bucket.js index 8e2668266f..6be184ac07 100644 --- a/storagetransfer/test/utils/bucket.js +++ b/storagetransfer/test/utils/bucket.js @@ -27,7 +27,6 @@ const uuid = require('uuid'); class BucketManager { constructor() { this.client = new StorageTransferServiceClient(); - this.s3 = new AWS.S3({apiVersion: '2006-03-01'}); this.storage = new Storage(); /** @@ -45,6 +44,10 @@ class BucketManager { this.s3Buckets = []; } + setupS3(options = {}) { + this.s3 = new AWS.S3({apiVersion: '2006-03-01', ...options}); + } + async getProjectId() { if (!this._cachedProjectId) { this._cachedProjectId = await this.storage.getProjectId(); From 6acab0076ad84fdd5a57f58a132016265a05b144 Mon Sep 17 00:00:00 2001 From: Daniel Bankhead Date: Thu, 15 Dec 2022 13:33:38 -0800 Subject: [PATCH 25/27] feat(samples): Add Azure Sample (#134) --- storagetransfer/azure-request.js | 110 +++++++++++++++++++++ storagetransfer/package.json | 1 + storagetransfer/test/azure-request.test.js | 83 ++++++++++++++++ storagetransfer/test/utils/bucket.js | 44 +++++++-- 4 files changed, 231 insertions(+), 7 deletions(-) create mode 100644 storagetransfer/azure-request.js create mode 100644 storagetransfer/test/azure-request.test.js diff --git a/storagetransfer/azure-request.js b/storagetransfer/azure-request.js new file mode 100644 index 0000000000..16e9bf10b4 --- /dev/null +++ b/storagetransfer/azure-request.js @@ -0,0 +1,110 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +async function main( + projectId, + description, + azureStorageAccount, + azureSourceContainer, + gcsSinkBucket, + azureSASToken = process.env.AZURE_SAS_TOKEN +) { + // [START storagetransfer_transfer_from_azure] + + // Imports the Google Cloud client library + const { + StorageTransferServiceClient, + } = require('@google-cloud/storage-transfer'); + + /** + * TODO(developer): Uncomment the following lines before running the sample. + */ + // The ID of the Google Cloud Platform Project that owns the job + // projectId = 'my-project-id' + + // A useful description for your transfer job + // description = 'My transfer job' + + // Azure Storage Account name + // azureStorageAccount = 'accountname' + + // Azure Storage Account name + // azureSourceContainer = 'my-azure-source-bucket' + + // Azure Shared Access Signature token + // azureSASToken = '?sv=...' + + // Google Cloud Storage destination bucket name + // gcsSinkBucket = 'my-gcs-destination-bucket' + + // Creates a client + const client = new StorageTransferServiceClient(); + + /** + * Creates a one-time transfer job from Azure Blob Storage to Google Cloud Storage. + */ + async function transferFromBlobStorage() { + // Setting the start date and the end date as the same time creates a + // one-time transfer + const now = new Date(); + const oneTimeSchedule = { + day: now.getDate(), + month: now.getMonth() + 1, + year: now.getFullYear(), + }; + + // Runs the request and creates the job + const [transferJob] = await client.createTransferJob({ + transferJob: { + projectId, + description, + status: 'ENABLED', + schedule: { + scheduleStartDate: oneTimeSchedule, + scheduleEndDate: oneTimeSchedule, + }, + transferSpec: { + azureBlobStorageDataSource: { + azureCredentials: { + sasToken: azureSASToken, + }, + container: azureSourceContainer, + storageAccount: azureStorageAccount, + }, + gcsDataSink: { + bucketName: gcsSinkBucket, + }, + }, + }, + }); + + console.log( + `Created and ran a transfer job from '${azureSourceContainer}' to '${gcsSinkBucket}' with name ${transferJob.name}` + ); + } + + transferFromBlobStorage(); + // [END storagetransfer_transfer_from_azure] +} + +main(...process.argv.slice(2)); + +process.on('unhandledRejection', err => { + console.error(err); + process.exitCode = 1; +}); diff --git a/storagetransfer/package.json b/storagetransfer/package.json index 47241a24b8..a54b6b8358 100644 --- a/storagetransfer/package.json +++ b/storagetransfer/package.json @@ -16,6 +16,7 @@ "@google-cloud/storage-transfer": "^2.2.1" }, "devDependencies": { + "@azure/storage-blob": "^12.12.0", "@google-cloud/storage": "^6.0.0", "aws-sdk": "^2.1073.0", "c8": "^7.1.0", diff --git a/storagetransfer/test/azure-request.test.js b/storagetransfer/test/azure-request.test.js new file mode 100644 index 0000000000..1d9570dd86 --- /dev/null +++ b/storagetransfer/test/azure-request.test.js @@ -0,0 +1,83 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const {assert} = require('chai'); +const {after, before, describe, it} = require('mocha'); + +const {BucketManager, TransferJobManager, runSample} = require('./utils'); + +describe('azure-request', () => { + const testBucketManager = new BucketManager(); + const testTransferJobManager = new TransferJobManager(); + + let projectId; + let description; + let azureStorageAccount; + let azureSourceContainer; + let gcsSinkBucket; + + before(async () => { + assert( + process.env.AZURE_CONNECTION_STRING, + 'environment variable AZURE_CONNECTION_STRING is required' + ); + + testBucketManager.setupBlobStorageFromConnectionString( + process.env.AZURE_CONNECTION_STRING + ); + + azureStorageAccount = + process.env.AZURE_STORAGE_ACCOUNT || + testBucketManager.blobStorage.accountName; + + projectId = await testBucketManager.getProjectId(); + azureSourceContainer = + await testBucketManager.generateBlobStorageContainer(); + gcsSinkBucket = (await testBucketManager.generateGCSBucket()).name; + description = `My transfer job from '${azureSourceContainer}' -> '${gcsSinkBucket}'`; + + if (!process.env.AZURE_SAS_TOKEN) { + // For security purposes we only want to pass this value via environment, not cli + process.env.AZURE_SAS_TOKEN = new URL( + testBucketManager.blobStorage.storageClientContext.url + ).search; + } + }); + + after(async () => { + await testBucketManager.deleteBuckets(); + await testTransferJobManager.cleanUp(); + }); + + it('should create a transfer job from Azure to GCS', async () => { + const output = await runSample('azure-request', [ + projectId, + description, + azureStorageAccount, + azureSourceContainer, + gcsSinkBucket, + ]); + + assert.include(output, 'Created and ran a transfer job'); + + // If it ran successfully and a job was created, delete it to clean up + const [jobName] = output.match(/transferJobs.*/); + + testTransferJobManager.transferJobToCleanUp(jobName); + }); +}); diff --git a/storagetransfer/test/utils/bucket.js b/storagetransfer/test/utils/bucket.js index 6be184ac07..d9c244d3aa 100644 --- a/storagetransfer/test/utils/bucket.js +++ b/storagetransfer/test/utils/bucket.js @@ -22,6 +22,7 @@ const { StorageTransferServiceClient, } = require('@google-cloud/storage-transfer'); const AWS = require('aws-sdk'); +const AzureStorageBlob = require('@azure/storage-blob'); const uuid = require('uuid'); class BucketManager { @@ -38,12 +39,21 @@ class BucketManager { * @type {Bucket[]} */ this.gcsBuckets = []; + /** + * @type {string[]} + */ + this.blobStorageContainers = []; /** * @type {string[]} */ this.s3Buckets = []; } + setupBlobStorageFromConnectionString(connectionString = '') { + this.blobStorage = + AzureStorageBlob.BlobServiceClient.fromConnectionString(connectionString); + } + setupS3(options = {}) { this.s3 = new AWS.S3({apiVersion: '2006-03-01', ...options}); } @@ -156,11 +166,9 @@ class BucketManager { * Configures STS read/write perms on the bucket. * * Is cached for easy clean-up via {#deleteBuckets}. - * - * @returns */ async generateGCSBucket() { - const name = await BucketManager.generateBucketName(); + const name = BucketManager.generateBucketName(); const bucket = this.storage.bucket(name); this.gcsBuckets.push(bucket); @@ -171,16 +179,30 @@ class BucketManager { } /** - * Generates a unique GCS bucket for testing. - * Configures STS read/write perms on the bucket. + * Generates a unique Azure container for testing. * * Is cached for easy clean-up via {#deleteBuckets}. + */ + async generateBlobStorageContainer() { + const name = BucketManager.generateBucketName(); + + // Create a container + const containerClient = this.blobStorage.getContainerClient(name); + await containerClient.create(); + + this.blobStorageContainers.push(name); + + return name; + } + + /** + * Generates a unique S3 bucket for testing. * - * @returns + * Is cached for easy clean-up via {#deleteBuckets}. */ async generateS3Bucket() { - const name = await BucketManager.generateBucketName(); + const name = BucketManager.generateBucketName(); await new Promise((resolve, reject) => { this.s3.createBucket({Bucket: name}, (error, data) => { @@ -207,6 +229,14 @@ class BucketManager { } } + for (const container of this.blobStorageContainers) { + try { + await this.blobStorage.deleteContainer(container); + } catch (e) { + console.error(e); + } + } + for (const bucket of this.s3Buckets) { try { await new Promise((resolve, reject) => { From 23e2c748cf9387a4aeb29e96c6038a026e3cf0d6 Mon Sep 17 00:00:00 2001 From: Karl Weinmeister Date: Wed, 25 Jan 2023 13:13:58 -0600 Subject: [PATCH 26/27] update workflow --- .github/workflows/storagetransfer.yaml | 82 ++++++++++++++++++++++++++ .github/workflows/workflows.json | 1 + CODEOWNERS | 5 +- 3 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/storagetransfer.yaml diff --git a/.github/workflows/storagetransfer.yaml b/.github/workflows/storagetransfer.yaml new file mode 100644 index 0000000000..7c0f2c19fc --- /dev/null +++ b/.github/workflows/storagetransfer.yaml @@ -0,0 +1,82 @@ +name: storagetransfer +on: + push: + branches: + - main + paths: + - 'storagetransfer/**' + - '.github/workflows/storagetransfer.yaml' + pull_request: + paths: + - 'storagetransfer/**' + - '.github/workflows/storagetransfer.yaml' + pull_request_target: + types: [labeled] + paths: + - 'storagetransfer/**' + - '.github/workflows/storagetransfer.yaml' + schedule: + - cron: '0 0 * * 0' +jobs: + test: + if: ${{ github.event.action != 'labeled' || github.event.label.name == 'actions:force-run' }} + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: 'write' + pull-requests: 'write' + id-token: 'write' + steps: + - uses: actions/checkout@v3.1.0 + with: + ref: ${{github.event.pull_request.head.sha}} + - uses: 'google-github-actions/auth@v1.0.0' + with: + workload_identity_provider: 'projects/1046198160504/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider' + service_account: 'kokoro-system-test@long-door-651.iam.gserviceaccount.com' + create_credentials_file: 'true' + access_token_lifetime: 600s + - id: secrets + uses: "google-github-actions/get-secretmanager-secrets@v1" + with: + secrets: |- + sts_aws_secret:cloud-devrel-kokoro-resources/nodejs-storagetransfer-azure + sts_azure_secret:cloud-devrel-kokoro-resources/nodejs-storagetransfer-azure + - uses: actions/setup-node@v3.5.1 + with: + node-version: 16 + - run: npm install + working-directory: storagetransfer + - run: npm test + working-directory: storagetransfer + env: + AWS_ACCESS_KEY_ID : ${{ fromJSON(steps.secrets.outputs.sts_aws_secret).AccessKeyId }} + AWS_SECRET_ACCESS_KEY: ${{ fromJSON(steps.secrets.outputs.sts_aws_secret).SecretAccessKey }} + AZURE_STORAGE_ACCOUNT: ${{ fromJSON(steps.secrets.outputs.sts_azure_secret).StorageAccount }} + AZURE_CONNECTION_STRING: ${{ fromJSON(steps.secrets.outputs.sts_azure_secret).ConnectionString }} + AZURE_SAS_TOKEN: ${{ fromJSON(steps.secrets.outputs.sts_azure_secret).SAS }} + MOCHA_REPORTER_SUITENAME: storagetransfer + MOCHA_REPORTER_OUTPUT: storagetransfer_sponge_log.xml + MOCHA_REPORTER: xunit + - if: ${{ github.event.action == 'labeled' && github.event.label.name == 'actions:force-run' }} + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + try { + await github.rest.issues.removeLabel({ + name: 'actions:force-run', + owner: 'GoogleCloudPlatform', + repo: 'nodejs-docs-samples', + issue_number: context.payload.pull_request.number + }); + } catch (e) { + if (!e.message.includes('Label does not exist')) { + throw e; + } + } + - if: ${{ github.event_name == 'schedule' && always() }} + run: | + curl https://github.com/googleapis/repo-automation-bots/releases/download/flakybot-1.1.0/flakybot -o flakybot -s -L + chmod +x ./flakybot + ./flakybot --repo GoogleCloudPlatform/nodejs-docs-samples --commit_hash ${{github.sha}} --build_url https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} diff --git a/.github/workflows/workflows.json b/.github/workflows/workflows.json index b224ef1bc2..83f33196b8 100644 --- a/.github/workflows/workflows.json +++ b/.github/workflows/workflows.json @@ -82,6 +82,7 @@ "service-directory/snippets", "secret-manager", "speech", + "storagetransfer", "talent", "texttospeech", "translate", diff --git a/CODEOWNERS b/CODEOWNERS index 750ba5cc87..1c36c90c65 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -47,8 +47,9 @@ functions/memorystore @GoogleCloudPlatform/nodejs-samples-reviewers functions/spanner @jsimonweb @GoogleCloudPlatform/nodejs-samples-reviewers # SoDa teams -/cloud-sql/**/*.js @GoogleCloudPlatform/infra-db-dpes @GoogleCloudPlatform/nodejs-samples-reviewers -/datastore/**/*.js @GoogleCloudPlatform/cloud-native-db-dpes @GoogleCloudPlatform/nodejs-samples-reviewers +cloud-sql @GoogleCloudPlatform/infra-db-dpes @GoogleCloudPlatform/nodejs-samples-reviewers +datastore @GoogleCloudPlatform/cloud-native-db-dpes @GoogleCloudPlatform/nodejs-samples-reviewers +storagetransfer @GoogleCloudPlatform/cloud-native-db-dpes @GoogleCloudPlatform/nodejs-samples-reviewers # One-offs composer @leahecole @sofisl @GoogleCloudPlatform/nodejs-samples-reviewers From ade31830263b1ce7737d481380923d2947b25be5 Mon Sep 17 00:00:00 2001 From: Karl Weinmeister Date: Wed, 25 Jan 2023 14:47:37 -0600 Subject: [PATCH 27/27] update workflow --- .github/workflows/storagetransfer.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/storagetransfer.yaml b/.github/workflows/storagetransfer.yaml index 7c0f2c19fc..33bcf594d9 100644 --- a/.github/workflows/storagetransfer.yaml +++ b/.github/workflows/storagetransfer.yaml @@ -40,8 +40,8 @@ jobs: uses: "google-github-actions/get-secretmanager-secrets@v1" with: secrets: |- - sts_aws_secret:cloud-devrel-kokoro-resources/nodejs-storagetransfer-azure - sts_azure_secret:cloud-devrel-kokoro-resources/nodejs-storagetransfer-azure + sts_aws_secret:nodejs-docs-samples-tests/nodejs-docs-samples-storagetransfer-aws + sts_azure_secret:nodejs-docs-samples-tests/nodejs-docs-samples-storagetransfer-azure - uses: actions/setup-node@v3.5.1 with: node-version: 16