diff --git a/.kokoro/run/image-processing.cfg b/.kokoro/run/image-processing.cfg new file mode 100644 index 0000000000..32ec554b93 --- /dev/null +++ b/.kokoro/run/image-processing.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Set the folder in which the tests are run +env_vars: { + key: "PROJECT" + value: "run/image-processing" +} diff --git a/run/README.md b/run/README.md index 393ed9c2b8..4e24c4c9e4 100644 --- a/run/README.md +++ b/run/README.md @@ -9,8 +9,9 @@ | Sample | Description | Deploy | | --------------------------------------- | ------------------------ | ------------- | |[Hello World][helloworld] ➥ | Quickstart | [Run on Google Cloud][run_button_helloworld] | +| [Image Processing][image_processing] | Event-driven image analysis & transformation | [Run on Google Cloud][run_button_image_processing] | |[Manual Logging][manual_logging] | Structured logging without client library | [Run on Google Cloud][run_button_manual_logging] | -|[Pub/Sub][pubsub] | Pub/Sub push Handler | [Run on Google Cloud][run_button_pubsub] | +|[Pub/Sub][pubsub] | Event-driven service with a Pub/Sub push subscription | [Run on Google Cloud][run_button_pubsub] | For more Cloud Run samples beyond Node.js, see the main list in the [Cloud Run Samples repository](https://github.com/GoogleCloudPlatform/cloud-run-samples). @@ -109,8 +110,10 @@ for more information. [run_build]: https://cloud.google.com/run/docs/building/containers [run_deploy]: https://cloud.google.com/run/docs/deploying [helloworld]: https://github.com/knative/docs/tree/master/docs/serving/samples/hello-world/helloworld-nodejs +[image_processing]: image-processing/ [manual_logging]: logging-manual/ [pubsub]: pubsub/ [run_button_helloworld]: https://console.cloud.google.com/cloudshell/editor?shellonly=true&cloudshell_image=gcr.io/cloudrun/button&cloudshell_git_repo=https://github.com/knative/docs&cloudshell_working_dir=docs/serving/samples/hello-world/helloworld-nodejs +[run_button_image_processing]: https://console.cloud.google.com/cloudshell/editor?shellonly=true&cloudshell_image=gcr.io/cloudrun/button&cloudshell_git_repo=https://github.com/GoogleCloudPlatform/nodejs-docs-samples&cloudshell_working_dir=run/image-processing [run_button_manual_logging]: https://console.cloud.google.com/cloudshell/editor?shellonly=true&cloudshell_image=gcr.io/cloudrun/button&cloudshell_git_repo=https://github.com/GoogleCloudPlatform/nodejs-docs-samples&cloudshell_working_dir=run/logging-manual [run_button_pubsub]: https://console.cloud.google.com/cloudshell/editor?shellonly=true&cloudshell_image=gcr.io/cloudrun/button&cloudshell_git_repo=https://github.com/GoogleCloudPlatform/nodejs-docs-samples&cloudshell_working_dir=run/pubsub \ No newline at end of file diff --git a/run/image-processing/.dockerignore b/run/image-processing/.dockerignore new file mode 100644 index 0000000000..5747c4c87d --- /dev/null +++ b/run/image-processing/.dockerignore @@ -0,0 +1,4 @@ +Dockerfile +.dockerignore +node_modules +npm-debug.log diff --git a/run/image-processing/.env b/run/image-processing/.env new file mode 100644 index 0000000000..b51008467a --- /dev/null +++ b/run/image-processing/.env @@ -0,0 +1,2 @@ +export INPUT_BUCKET_NAME=adamross-svls-kibble +export BLURRED_BUCKET_NAME=adamross-svls-kibble-output diff --git a/run/image-processing/.gcloudignore b/run/image-processing/.gcloudignore new file mode 100644 index 0000000000..26600c93a8 --- /dev/null +++ b/run/image-processing/.gcloudignore @@ -0,0 +1,3 @@ +.gcloudignore +node_modules +npm-debug.log diff --git a/run/image-processing/.gitignore b/run/image-processing/.gitignore new file mode 100644 index 0000000000..3c3629e647 --- /dev/null +++ b/run/image-processing/.gitignore @@ -0,0 +1 @@ +node_modules diff --git a/run/image-processing/Dockerfile b/run/image-processing/Dockerfile new file mode 100644 index 0000000000..70a617b235 --- /dev/null +++ b/run/image-processing/Dockerfile @@ -0,0 +1,38 @@ +# Copyright 2019 Google LLC. All rights reserved. +# Use of this source code is governed by the Apache 2.0 +# license that can be found in the LICENSE file. + +# Use the official Node.js 10 image. +# https://hub.docker.com/_/node +FROM node:10 + +# [START run_imageproc_dockerfile_imagemagick] + +# Install Imagemagick into the container image. +# For more on system packages review the system packages tutorial. +# https://cloud.google.com/run/docs/tutorials/system-packages#dockerfile +RUN set -ex; \ + apt-get -y update; \ + apt-get -y install imagemagick; \ + rm -rf /var/lib/apt/lists/* + +# [END run_imageproc_dockerfile_imagemagick] + +# Create and change to the app directory. +WORKDIR /usr/src/app + +# Copy application dependency manifests to the container image. +# A wildcard is used to ensure both package.json AND package-lock.json are copied. +# Copying this separately prevents re-running npm install on every code change. +COPY package*.json ./ + +# Install dependencies. +RUN npm install --production +# If you add a package-lock.json, speed your build by switching to 'npm ci'. +# RUN npm ci --only=production + +# Copy local code to the container image. +COPY . . + +# Run the web service on container startup. +CMD [ "npm", "start" ] diff --git a/run/image-processing/README.md b/run/image-processing/README.md new file mode 100644 index 0000000000..63b44c3b39 --- /dev/null +++ b/run/image-processing/README.md @@ -0,0 +1,32 @@ +# Cloud Run Image Processing Sample + +This sample service applies [Cloud Storage](https://cloud.google.com/storage/docs)-triggered image processing with [Cloud Vision API](https://cloud.google.com/vision/docs) analysis and ImageMagick transformation. + +Use it with the [Image Processing with Cloud Run tutorial](http://cloud.google.com/run/docs/tutorials/image-processing). + +For more details on how to work with this sample read the [Google Cloud Run Node.js Samples README](https://github.com/GoogleCloudPlatform/nodejs-docs-samples/run). + +## Dependencies + +* **express**: Web server framework +* **body-parser**: express middleware for request payload processing + +## Environment Variables + +Cloud Run services can be [configured with Environment Variables](https://cloud.google.com/run/docs/configuring/environment-variables). +Required variables for this sample include: + +* `INPUT_BUCKET_NAME`: The Cloud Run service will be notified of images uploaded to this Cloud Storage bucket. The service will then retreive and process the image. +* `BLURRED_BUCKET_NAME`: The Cloud Run service will write blurred images to this Cloud Storage bucket. + +## Maintenance Note + +* The `image.js` file is copied from the [Cloud Functions ImageMagick sample `index.js`](../../functions/imagemagick/index.js). Region tags are changed. +* The package.json dependencies used in the copied code should track the [Cloud Functions ImageMagick `package.json`](../../functions/imagemagick/package.json) + +```sh +cp ../../functions/imagemagick/index.js image.js +sed -i '' 's/functions_imagemagick_setup/run_imageproc_handler_setup/' image.js +sed -i '' 's/functions_imagemagick_analyze/run_imageproc_handler_analyze/' image.js +sed -i '' 's/functions_imagemagick_blur/run_imageproc_handler_blur/' image.js +``` diff --git a/run/image-processing/app.js b/run/image-processing/app.js new file mode 100644 index 0000000000..574b153535 --- /dev/null +++ b/run/image-processing/app.js @@ -0,0 +1,64 @@ +// Copyright 2019 Google LLC. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +// [START run_imageproc_controller] + +const express = require('express'); +const bodyParser = require('body-parser'); +const app = express(); + +app.use(bodyParser.json()); + +const image = require('./image'); + +app.post('/', async (req, res) => { + if (!req.body) { + const msg = 'no Pub/Sub message received'; + console.error(`error: ${msg}`); + res.status(400).send(`Bad Request: ${msg}`); + return; + } + if (!req.body.message || !req.body.message.data) { + const msg = 'invalid Pub/Sub message format'; + console.error(`error: ${msg}`); + res.status(400).send(`Bad Request: ${msg}`); + return; + } + + // Decode the Pub/Sub message. + const pubSubMessage = req.body.message; + let data; + try { + data = Buffer.from(pubSubMessage.data, 'base64') + .toString() + .trim(); + data = JSON.parse(data); + } catch (err) { + const msg = + 'Invalid Pub/Sub message: data property is not valid base64 encoded JSON'; + console.error(`error: ${msg}: ${err}`); + res.status(400).send(`Bad Request: ${msg}`); + return; + } + + // Validate the message is a Cloud Storage event. + if (!data.name || !data.bucket) { + const msg = + 'invalid Cloud Storage notification: expected name and bucket properties'; + console.error(`error: ${msg}`); + res.status(400).send(`Bad Request: ${msg}`); + return; + } + + try { + await image.blurOffensiveImages(data); + res.status(204).send(); + } catch (err) { + console.error(`error: Blurring image: ${err}`); + res.status(500).send(); + } +}); +// [END run_pubsub_handler] + +module.exports = app; diff --git a/run/image-processing/image.js b/run/image-processing/image.js new file mode 100644 index 0000000000..f8a41c8770 --- /dev/null +++ b/run/image-processing/image.js @@ -0,0 +1,108 @@ +/** + * Copyright 2017, Google, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +// [START run_imageproc_handler_setup] +const gm = require('gm').subClass({imageMagick: true}); +const fs = require('fs'); +const {promisify} = require('util'); +const path = require('path'); +const vision = require('@google-cloud/vision'); + +const {Storage} = require('@google-cloud/storage'); +const storage = new Storage(); +const client = new vision.ImageAnnotatorClient(); + +const {BLURRED_BUCKET_NAME} = process.env; +// [END run_imageproc_handler_setup] + +// [START run_imageproc_handler_analyze] +// Blurs uploaded images that are flagged as Adult or Violence. +exports.blurOffensiveImages = async event => { + // This event represents the triggering Cloud Storage object. + const object = event; + + const file = storage.bucket(object.bucket).file(object.name); + const filePath = `gs://${object.bucket}/${object.name}`; + + console.log(`Analyzing ${file.name}.`); + + try { + const [result] = await client.safeSearchDetection(filePath); + const detections = result.safeSearchAnnotation || {}; + + if ( + // Levels are defined in https://cloud.google.com/vision/docs/reference/rest/v1/AnnotateImageResponse#likelihood + detections.adult === 'VERY_LIKELY' || + detections.violence === 'VERY_LIKELY' + ) { + console.log(`Detected ${file.name} as inappropriate.`); + return blurImage(file, BLURRED_BUCKET_NAME); + } else { + console.log(`Detected ${file.name} as OK.`); + } + } catch (err) { + console.error(`Failed to analyze ${file.name}.`, err); + throw err; + } +}; +// [END run_imageproc_handler_analyze] + +// [START run_imageproc_handler_blur] +// Blurs the given file using ImageMagick, and uploads it to another bucket. +const blurImage = async (file, blurredBucketName) => { + const tempLocalPath = `/tmp/${path.parse(file.name).base}`; + + // Download file from bucket. + try { + await file.download({destination: tempLocalPath}); + + console.log(`Downloaded ${file.name} to ${tempLocalPath}.`); + } catch (err) { + throw new Error(`File download failed: ${err}`); + } + + await new Promise((resolve, reject) => { + gm(tempLocalPath) + .blur(0, 16) + .write(tempLocalPath, (err, stdout) => { + if (err) { + console.error('Failed to blur image.', err); + reject(err); + } else { + console.log(`Blurred image: ${file.name}`); + resolve(stdout); + } + }); + }); + + // Upload result to a different bucket, to avoid re-triggering this function. + const blurredBucket = storage.bucket(blurredBucketName); + + // Upload the Blurred image back into the bucket. + const gcsPath = `gs://${blurredBucketName}/${file.name}`; + try { + await blurredBucket.upload(tempLocalPath, {destination: file.name}); + console.log(`Uploaded blurred image to: ${gcsPath}`); + } catch (err) { + throw new Error(`Unable to upload blurred image to ${gcsPath}: ${err}`); + } + + // Delete the temporary file. + const unlink = promisify(fs.unlink); + return unlink(tempLocalPath); +}; +// [END run_imageproc_handler_blur] diff --git a/run/image-processing/index.js b/run/image-processing/index.js new file mode 100644 index 0000000000..7a7d33042e --- /dev/null +++ b/run/image-processing/index.js @@ -0,0 +1,12 @@ +// Copyright 2019 Google LLC. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +// [START run_imageproc_server] +const app = require('./app.js'); +const PORT = process.env.PORT || 8080; + +app.listen(PORT, () => + console.log(`nodejs-image-processing listening on port ${PORT}`) +); +// [END run_imageproc_server] diff --git a/run/image-processing/package.json b/run/image-processing/package.json new file mode 100644 index 0000000000..48e9837d33 --- /dev/null +++ b/run/image-processing/package.json @@ -0,0 +1,31 @@ +{ + "name": "nodejs-image-processing", + "version": "1.0.0", + "private": true, + "description": "Cloud Storage-triggered image processing with Cloud Vision API and ImageMagick transformation.", + "main": "index.js", + "author": "Google LLC", + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "https://github.com/GoogleCloudPlatform/nodejs-docs-samples.git" + }, + "engines": { + "node": ">= 8.0.0" + }, + "scripts": { + "start": "node index.js", + "test": "mocha test/*.test.js --check-leaks --timeout=5000" + }, + "dependencies": { + "@google-cloud/storage": "^3.0.4", + "@google-cloud/vision": "^1.1.3", + "body-parser": "^1.19.0", + "express": "^4.16.4", + "gm": "^1.23.1" + }, + "devDependencies": { + "mocha": "^6.1.4", + "supertest": "^4.0.2" + } +} diff --git a/run/image-processing/test/app.test.js b/run/image-processing/test/app.test.js new file mode 100644 index 0000000000..2e5f69dfb1 --- /dev/null +++ b/run/image-processing/test/app.test.js @@ -0,0 +1,101 @@ +const path = require('path'); +const supertest = require('supertest'); + +let request; + +describe('Unit Tests', () => { + before(() => { + const app = require(path.join(__dirname, '..', 'app')); + request = supertest(app); + }); + + describe('should fail', () => { + it(`on a Bad Request with an empty payload`, async () => { + await request + .post('/') + .type('json') + .send('') + .expect(400); + }); + + it(`on a Bad Request with an invalid payload`, async () => { + await request + .post('/') + .type('json') + .send({nomessage: 'invalid'}) + .expect(400); + }); + + it(`on a Bad Request with an invalid mimetype`, async () => { + await request + .post('/') + .type('text') + .send('{message: true}') + .expect(400); + }); + + it(`if the decoded message.data is not well-formed JSON`, async () => { + await request + .post('/') + .type('json') + .send({ + message: { + data: 'non-JSON value', + }, + }) + .expect(400); + }); + + describe('if name or bucket is missing from message.data, including', () => { + it('missing both "name" and "bucket"', async () => { + await request + .post('/') + .type('json') + .send({ + message: { + data: Buffer.from('{ "json": "value" }').toString(`base64`), + }, + }) + .expect(400); + }); + it('missing just the "name" property', async () => { + await request + .post('/') + .type('json') + .send({ + message: { + data: Buffer.from('{ "name": "value" }').toString(`base64`), + }, + }) + .expect(400); + }); + it('missing just the "bucket" property', async () => { + await request + .post('/') + .type('json') + .send({ + message: { + data: Buffer.from('{ "bucket": "value" }').toString(`base64`), + }, + }) + .expect(400); + }); + }); + }); +}); + +describe('Integration Tests', () => { + it(`Image analysis can proceed to Vision API scan`, async () => { + await request + .post('/') + .type('json') + .send({ + message: { + data: Buffer.from( + `{ "bucket": "---", "name": "does-not-exist" }` + ).toString(`base64`), + }, + }) + .expect(204); + }); +}); diff --git a/run/pubsub/README.md b/run/pubsub/README.md index ab385ca4cb..fadedf4fdf 100644 --- a/run/pubsub/README.md +++ b/run/pubsub/README.md @@ -10,4 +10,6 @@ For more details on how to work with this sample read the [Google Cloud Run Node * **express**: Web server framework. * **body-parser**: express middleware for request payload processing. - +* **[gm](https://github.com/aheckmann/gm#readme)**: ImageMagick integration library. +* **@google-cloud/storage**: Google Cloud Storage client library. +* **@google-cloud/vision**: Cloud Vision API client library. \ No newline at end of file