From 2b7aadcc5ff1827c0797b5fa745d77cba9d56a57 Mon Sep 17 00:00:00 2001 From: Sofia Leon Date: Mon, 6 Apr 2020 18:12:24 -0700 Subject: [PATCH 1/6] feat: add node sample for cloud compser tutorial --- composer/composerJSTutorial.js | 46 ++++++++++++++++++++++++ composer/package.json | 22 ++++++++++++ composer/test/composerJSTutorial.test.js | 15 ++++++++ 3 files changed, 83 insertions(+) create mode 100644 composer/composerJSTutorial.js create mode 100644 composer/package.json create mode 100644 composer/test/composerJSTutorial.test.js diff --git a/composer/composerJSTutorial.js b/composer/composerJSTutorial.js new file mode 100644 index 0000000000..f44806d813 --- /dev/null +++ b/composer/composerJSTutorial.js @@ -0,0 +1,46 @@ +/* eslint-disable func-style */ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +module.exports = function main( + line = 'tampa, 106, january, null, null, 08--17-2019' +) { + // [START transform_CSV_to_JSON] + + function transformCSVtoJSON(line) { + const values = line.split(','); + const properties = [ + 'location', + 'average_temperature', + 'month', + 'inches_of_rain', + 'is_current', + 'latest_measurement', + ]; + const weatherInCity = {}; + + for (let count = 0; count < values.length; count++) { + if (values[count] !== 'null') { + weatherInCity[properties[count]] = values[count]; + } + } + + const jsonString = JSON.stringify(weatherInCity); + return jsonString; + } + + transformCSVtoJSON(line); + // [END transform_CSV_to_JSON] + return transformCSVtoJSON(line); +}; diff --git a/composer/package.json b/composer/package.json new file mode 100644 index 0000000000..4c7fd51ee6 --- /dev/null +++ b/composer/package.json @@ -0,0 +1,22 @@ +{ + "name": "nodejs-docs-samples-composer", + "description": "Node.js Google Composer sample.", + "version": "0.0.1", + "private": true, + "license": "Apache-2.0", + "author": "Google Inc.", + "repository": { + "type": "git", + "url": "https://github.com/GoogleCloudPlatform/nodejs-docs-samples.git" + }, + "engines": { + "node": ">=8.0.0" + }, + "scripts": { + "test": "mocha --exit test/*.test.js" + }, + "devDependencies": { + "chai": "^4.2.0", + "mocha": "^7.0.0" + } +} diff --git a/composer/test/composerJSTutorial.test.js b/composer/test/composerJSTutorial.test.js new file mode 100644 index 0000000000..5b3436ed4b --- /dev/null +++ b/composer/test/composerJSTutorial.test.js @@ -0,0 +1,15 @@ +const {assert} = require('chai'); +const tutorial = require('../composerJSTutorial.js'); + +const lineTest = 'tampa, 106, january, null, null, 08-17-2019'; + +it('should separate out a line and turn into a json string', () => { + const output = tutorial(lineTest); + console.log(output); + assert.match( + output, + new RegExp( + '{"location":"tampa","average_temperature":" 106","month":" january","inches_of_rain":" null","is_current":" null","latest_measurement":" 08-17-2019"}' + ) + ); +}); From 9d8a9681439d69e31a5e26b3da17264061692090 Mon Sep 17 00:00:00 2001 From: Sofia Leon Date: Tue, 7 Apr 2020 15:23:30 -0700 Subject: [PATCH 2/6] fix: respond to Leah comments --- CODEOWNERS | 4 ++++ ...poserJSTutorial.js => composer_transform_csv_to_json.js} | 6 +++--- ...orial.test.js => composer_transform_csv_to_json.test.js} | 1 - 3 files changed, 7 insertions(+), 4 deletions(-) rename composer/{composerJSTutorial.js => composer_transform_csv_to_json.js} (89%) rename composer/test/{composerJSTutorial.test.js => composer_transform_csv_to_json.test.js} (95%) diff --git a/CODEOWNERS b/CODEOWNERS index 7e35ae8036..056a0c892c 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -30,3 +30,7 @@ cloud-sql @kurtisvg healthcare @noerog iot @gguuss jobs @happyhuman + +#Composer samples +@sofisl +@leahecole \ No newline at end of file diff --git a/composer/composerJSTutorial.js b/composer/composer_transform_csv_to_json.js similarity index 89% rename from composer/composerJSTutorial.js rename to composer/composer_transform_csv_to_json.js index f44806d813..9e44b3bc79 100644 --- a/composer/composerJSTutorial.js +++ b/composer/composer_transform_csv_to_json.js @@ -14,9 +14,9 @@ // limitations under the License. module.exports = function main( - line = 'tampa, 106, january, null, null, 08--17-2019' + line = 'tampa, 106, january, null, null, 08-17-2019' ) { - // [START transform_CSV_to_JSON] + // [START composer_transform_csv_to_json] function transformCSVtoJSON(line) { const values = line.split(','); @@ -41,6 +41,6 @@ module.exports = function main( } transformCSVtoJSON(line); - // [END transform_CSV_to_JSON] + // [END composer_transform_csv_to_json] return transformCSVtoJSON(line); }; diff --git a/composer/test/composerJSTutorial.test.js b/composer/test/composer_transform_csv_to_json.test.js similarity index 95% rename from composer/test/composerJSTutorial.test.js rename to composer/test/composer_transform_csv_to_json.test.js index 5b3436ed4b..24947939a5 100644 --- a/composer/test/composerJSTutorial.test.js +++ b/composer/test/composer_transform_csv_to_json.test.js @@ -5,7 +5,6 @@ const lineTest = 'tampa, 106, january, null, null, 08-17-2019'; it('should separate out a line and turn into a json string', () => { const output = tutorial(lineTest); - console.log(output); assert.match( output, new RegExp( From 8e0bd93ffffd78b421afb2d58ba57c0e06fa3958 Mon Sep 17 00:00:00 2001 From: Sofia Leon Date: Tue, 7 Apr 2020 15:41:20 -0700 Subject: [PATCH 3/6] feat: add README --- composer/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 composer/README.md diff --git a/composer/README.md b/composer/README.md new file mode 100644 index 0000000000..fb97cccb33 --- /dev/null +++ b/composer/README.md @@ -0,0 +1,9 @@ +# [Node sample for Cloud Composer-Dataflow Tutorial][tutorial-link] + +This is the code for the transform CSV-to-JSON code sample for the [Using the DataflowTemplateOperator][tutorial-link] tutorial. + +This tutorial how to use the DataflowTemplateOperator to launch Dataflow Pipelines from Cloud Composer. The Cloud Storage Text to BigQuery (Stream) pipeline is a streaming pipeline that allows you to stream text files stored in Cloud Storage, transform them using a JavaScript User Defined Function (UDF) that you provide, and output the results to BigQuery. + +* This code is a User Defined Function written in JavaScript that will transform each line of a .txt file into the relevant columns for a BigQuery table. + +[tutorial-link]: https://cloud.devsite.corp.google.com/composer/docs/how-to/using/using-dataflow-template-operator?auto_signin=false From e5380f3fa2f2a5443b112d4e5b2c6b54f2e51b8d Mon Sep 17 00:00:00 2001 From: Sofia Leon Date: Wed, 8 Apr 2020 14:43:30 -0700 Subject: [PATCH 4/6] fix: readme --- composer/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/composer/README.md b/composer/README.md index fb97cccb33..248a1df357 100644 --- a/composer/README.md +++ b/composer/README.md @@ -2,8 +2,8 @@ This is the code for the transform CSV-to-JSON code sample for the [Using the DataflowTemplateOperator][tutorial-link] tutorial. -This tutorial how to use the DataflowTemplateOperator to launch Dataflow Pipelines from Cloud Composer. The Cloud Storage Text to BigQuery (Stream) pipeline is a streaming pipeline that allows you to stream text files stored in Cloud Storage, transform them using a JavaScript User Defined Function (UDF) that you provide, and output the results to BigQuery. +This tutorial shows how to use the DataflowTemplateOperator to launch Dataflow Pipelines from Cloud Composer. The Cloud Storage Text to BigQuery (Stream) pipeline is a streaming pipeline that allows you to stream text files stored in Cloud Storage, transform them using a JavaScript User Defined Function (UDF) that you provide, and output the results to BigQuery. We'll use Cloud Composer to kick off a Cloud Dataflow pipeline that will apply the User-Defined Function to our .txt file and format it according to the JSON schema. -* This code is a User Defined Function written in JavaScript that will transform each line of a .txt file into the relevant columns for a BigQuery table. +* The code held in this repo is a User Defined Function written in JavaScript that will transform each line of a .txt file into the relevant columns for a BigQuery table. [tutorial-link]: https://cloud.devsite.corp.google.com/composer/docs/how-to/using/using-dataflow-template-operator?auto_signin=false From fceb9a9b4a64201272f49ce8f9052c2832d3e2dd Mon Sep 17 00:00:00 2001 From: Sofia Leon Date: Fri, 10 Apr 2020 15:58:19 -0700 Subject: [PATCH 5/6] fix: change readme and codeowners file --- composer/README.md | 2 +- composer/test/composer_transform_csv_to_json.test.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/composer/README.md b/composer/README.md index 248a1df357..ecb60ac351 100644 --- a/composer/README.md +++ b/composer/README.md @@ -4,6 +4,6 @@ This is the code for the transform CSV-to-JSON code sample for the [Using the Da This tutorial shows how to use the DataflowTemplateOperator to launch Dataflow Pipelines from Cloud Composer. The Cloud Storage Text to BigQuery (Stream) pipeline is a streaming pipeline that allows you to stream text files stored in Cloud Storage, transform them using a JavaScript User Defined Function (UDF) that you provide, and output the results to BigQuery. We'll use Cloud Composer to kick off a Cloud Dataflow pipeline that will apply the User-Defined Function to our .txt file and format it according to the JSON schema. -* The code held in this repo is a User Defined Function written in JavaScript that will transform each line of a .txt file into the relevant columns for a BigQuery table. +The code held in this repo is a User Defined Function written in JavaScript that will transform each line of a .txt file into the relevant columns for a BigQuery table. [tutorial-link]: https://cloud.devsite.corp.google.com/composer/docs/how-to/using/using-dataflow-template-operator?auto_signin=false diff --git a/composer/test/composer_transform_csv_to_json.test.js b/composer/test/composer_transform_csv_to_json.test.js index 24947939a5..356c016b47 100644 --- a/composer/test/composer_transform_csv_to_json.test.js +++ b/composer/test/composer_transform_csv_to_json.test.js @@ -1,5 +1,5 @@ const {assert} = require('chai'); -const tutorial = require('../composerJSTutorial.js'); +const tutorial = require('../composer_transform_csv_to_json.js'); const lineTest = 'tampa, 106, january, null, null, 08-17-2019'; From ad7ac34f5a36c5dc1a79718d0d81be982ff17abc Mon Sep 17 00:00:00 2001 From: Sofia Leon Date: Tue, 14 Apr 2020 21:26:48 -0700 Subject: [PATCH 6/6] fix: add owners in alphabetical order --- CODEOWNERS | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CODEOWNERS b/CODEOWNERS index 056a0c892c..a7b461f303 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -27,10 +27,8 @@ datastore/functions @benwhitehead # One-offs cloud-sql @kurtisvg +composer @leahecole @sofisl healthcare @noerog iot @gguuss jobs @happyhuman -#Composer samples -@sofisl -@leahecole \ No newline at end of file