diff --git a/CODEOWNERS b/CODEOWNERS index 7e35ae8036..a7b461f303 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -27,6 +27,8 @@ datastore/functions @benwhitehead # One-offs cloud-sql @kurtisvg +composer @leahecole @sofisl healthcare @noerog iot @gguuss jobs @happyhuman + diff --git a/composer/README.md b/composer/README.md new file mode 100644 index 0000000000..ecb60ac351 --- /dev/null +++ b/composer/README.md @@ -0,0 +1,9 @@ +# [Node sample for Cloud Composer-Dataflow Tutorial][tutorial-link] + +This is the code for the transform CSV-to-JSON code sample for the [Using the DataflowTemplateOperator][tutorial-link] tutorial. + +This tutorial shows how to use the DataflowTemplateOperator to launch Dataflow Pipelines from Cloud Composer. The Cloud Storage Text to BigQuery (Stream) pipeline is a streaming pipeline that allows you to stream text files stored in Cloud Storage, transform them using a JavaScript User Defined Function (UDF) that you provide, and output the results to BigQuery. We'll use Cloud Composer to kick off a Cloud Dataflow pipeline that will apply the User-Defined Function to our .txt file and format it according to the JSON schema. + +The code held in this repo is a User Defined Function written in JavaScript that will transform each line of a .txt file into the relevant columns for a BigQuery table. + +[tutorial-link]: https://cloud.devsite.corp.google.com/composer/docs/how-to/using/using-dataflow-template-operator?auto_signin=false diff --git a/composer/composer_transform_csv_to_json.js b/composer/composer_transform_csv_to_json.js new file mode 100644 index 0000000000..9e44b3bc79 --- /dev/null +++ b/composer/composer_transform_csv_to_json.js @@ -0,0 +1,46 @@ +/* eslint-disable func-style */ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +module.exports = function main( + line = 'tampa, 106, january, null, null, 08-17-2019' +) { + // [START composer_transform_csv_to_json] + + function transformCSVtoJSON(line) { + const values = line.split(','); + const properties = [ + 'location', + 'average_temperature', + 'month', + 'inches_of_rain', + 'is_current', + 'latest_measurement', + ]; + const weatherInCity = {}; + + for (let count = 0; count < values.length; count++) { + if (values[count] !== 'null') { + weatherInCity[properties[count]] = values[count]; + } + } + + const jsonString = JSON.stringify(weatherInCity); + return jsonString; + } + + transformCSVtoJSON(line); + // [END composer_transform_csv_to_json] + return transformCSVtoJSON(line); +}; diff --git a/composer/package.json b/composer/package.json new file mode 100644 index 0000000000..4c7fd51ee6 --- /dev/null +++ b/composer/package.json @@ -0,0 +1,22 @@ +{ + "name": "nodejs-docs-samples-composer", + "description": "Node.js Google Composer sample.", + "version": "0.0.1", + "private": true, + "license": "Apache-2.0", + "author": "Google Inc.", + "repository": { + "type": "git", + "url": "https://github.com/GoogleCloudPlatform/nodejs-docs-samples.git" + }, + "engines": { + "node": ">=8.0.0" + }, + "scripts": { + "test": "mocha --exit test/*.test.js" + }, + "devDependencies": { + "chai": "^4.2.0", + "mocha": "^7.0.0" + } +} diff --git a/composer/test/composer_transform_csv_to_json.test.js b/composer/test/composer_transform_csv_to_json.test.js new file mode 100644 index 0000000000..356c016b47 --- /dev/null +++ b/composer/test/composer_transform_csv_to_json.test.js @@ -0,0 +1,14 @@ +const {assert} = require('chai'); +const tutorial = require('../composer_transform_csv_to_json.js'); + +const lineTest = 'tampa, 106, january, null, null, 08-17-2019'; + +it('should separate out a line and turn into a json string', () => { + const output = tutorial(lineTest); + assert.match( + output, + new RegExp( + '{"location":"tampa","average_temperature":" 106","month":" january","inches_of_rain":" null","is_current":" null","latest_measurement":" 08-17-2019"}' + ) + ); +});