From 3b3db6443c04e2763f6927080e568c4f90a65a80 Mon Sep 17 00:00:00 2001 From: ace-n Date: Thu, 6 Jun 2019 16:08:57 -0700 Subject: [PATCH 1/3] ImageMagick: use separate bucket for blurred images. --- functions/imagemagick/README.md | 14 +++++++++++--- functions/imagemagick/index.js | 24 ++++++++++++++++-------- functions/imagemagick/test/index.test.js | 23 ++++++++++------------- 3 files changed, 37 insertions(+), 24 deletions(-) diff --git a/functions/imagemagick/README.md b/functions/imagemagick/README.md index af7b31c209..2bf7aa0f59 100644 --- a/functions/imagemagick/README.md +++ b/functions/imagemagick/README.md @@ -21,15 +21,23 @@ Functions for your project. 1. Create a Cloud Storage Bucket: - gsutil mb gs://YOUR_BUCKET_NAME + gsutil mb gs://YOUR_INPUT_BUCKET_NAME This storage bucket is used to upload images for the function to check. +1. Create a second Cloud Storage Bucket: + + gsutil mb gs://YOUR_OUTPUT_BUCKET_NAME + + This second storage bucket is used to store blurred images. (Un-blurred images will not be saved to this bucket.) + + This is necessary because saving the blurred image to the input bucket would cause your function to be invoked a second time with the blurred image itself. + 1. Deploy the `blurOffensiveImages` function with a Storage trigger: - gcloud functions deploy blurOffensiveImages --trigger-bucket=YOUR_BUCKET_NAME + gcloud functions deploy blurOffensiveImages --trigger-bucket=YOUR_INPUT_BUCKET_NAME --set-env-vars BLURRED_BUCKET_NAME=YOUR_OUTPUT_BUCKET_NAME - * Replace `YOUR_BUCKET_NAME` with the name of the Cloud Storage Bucket you created earlier. + * Replace `YOUR_INPUT_BUCKET_NAME` and `YOUR_OUTPUT_BUCKET_NAME` with the names of the respective Cloud Storage Buckets you created earlier. 1. Upload an offensive image to the Storage bucket, such as this image of a flesh-eating zombie: https://cdn.pixabay.com/photo/2015/09/21/14/24/zombie-949916_1280.jpg diff --git a/functions/imagemagick/index.js b/functions/imagemagick/index.js index a174e58047..d8b6658ba1 100644 --- a/functions/imagemagick/index.js +++ b/functions/imagemagick/index.js @@ -24,6 +24,8 @@ const storage = new Storage(); const vision = require('@google-cloud/vision').v1p1beta1; const client = new vision.ImageAnnotatorClient(); + +const {BLURRED_BUCKET_NAME} = process.env; // [END functions_imagemagick_setup] // [START functions_imagemagick_analyze] @@ -67,7 +69,7 @@ exports.blurOffensiveImages = event => { console.log( `The image ${file.name} has been detected as inappropriate.` ); - return blurImage(file); + return blurImage(file, BLURRED_BUCKET_NAME); } else { console.log(`The image ${file.name} has been detected as OK.`); } @@ -76,8 +78,8 @@ exports.blurOffensiveImages = event => { // [END functions_imagemagick_analyze] // [START functions_imagemagick_blur] -// Blurs the given file using ImageMagick. -function blurImage(file) { +// Blurs the given file using ImageMagick, and uploads it to another bucket. +function blurImage(file, blurredBucketName) { const tempLocalPath = `/tmp/${path.parse(file.name).base}`; // Download file from bucket. @@ -109,19 +111,25 @@ function blurImage(file) { .then(() => { console.log(`Image ${file.name} has been blurred.`); - // Mark result as blurred, to avoid re-triggering this function. - const newName = `blurred-${file.name}`; + // Upload result to a different bucket, to avoid re-triggering this function. + // You can also re-upload it to the same bucket + tell your Cloud Function to + // ignore files marked as blurred (e.g. those with a "blurred" prefix) + const blurredBucket = storage.bucket(blurredBucketName); // Upload the Blurred image back into the bucket. - return file.bucket - .upload(tempLocalPath, {destination: newName}) + return blurredBucket + .upload(tempLocalPath, {destination: file.name}) .catch(err => { console.error('Failed to upload blurred image.', err); return Promise.reject(err); }); }) .then(() => { - console.log(`Blurred image has been uploaded to ${file.name}.`); + console.log( + `Blurred image has been uploaded to: gs://${blurredBucketName}/${ + file.name + }` + ); // Delete the temporary file. return new Promise((resolve, reject) => { diff --git a/functions/imagemagick/test/index.test.js b/functions/imagemagick/test/index.test.js index 2769843387..911df164fc 100644 --- a/functions/imagemagick/test/index.test.js +++ b/functions/imagemagick/test/index.test.js @@ -22,8 +22,11 @@ const tools = require('@google-cloud/nodejs-repo-tools'); const vision = require('@google-cloud/vision').v1p1beta1; const bucketName = 'my-bucket'; +const blurredBucketName = 'my-blurred-bucket'; const defaultFileName = 'image.jpg'; +process.env.BLURRED_BUCKET_NAME = blurredBucketName; + let VisionStub = sinon.stub(vision, 'ImageAnnotatorClient'); VisionStub.returns({ safeSearchDetection: sinon.stub().returns( @@ -124,12 +127,15 @@ it('blurOffensiveImages blurs unblurred images (Node 6 syntax)', async () => { `Image ${sample.mocks.file.name} has been blurred.`, ]); assert.deepStrictEqual(console.log.getCall(4).args, [ - `Blurred image has been uploaded to ${sample.mocks.file.name}.`, + `Blurred image has been uploaded to: gs://${blurredBucketName}/${ + sample.mocks.file.name + }`, ]); }); it('blurOffensiveImages blurs unblurred images (Node 8 syntax)', async () => { const sample = getSample(defaultFileName); + await sample.program.blurOffensiveImages({ bucket: bucketName, name: defaultFileName, @@ -150,18 +156,9 @@ it('blurOffensiveImages blurs unblurred images (Node 8 syntax)', async () => { `Image ${sample.mocks.file.name} has been blurred.`, ]); assert.deepStrictEqual(console.log.getCall(4).args, [ - `Blurred image has been uploaded to ${sample.mocks.file.name}.`, - ]); -}); - -it('blurOffensiveImages ignores already-blurred images', async () => { - const sample = getSample('blurred-${defaultFileName}'); - await sample.program.blurOffensiveImages({ - data: {bucket: bucketName, name: `blurred-${defaultFileName}`}, - }); - assert.strictEqual(console.log.callCount, 1); - assert.deepStrictEqual(console.log.getCall(0).args, [ - `The image ${sample.mocks.file.name} is already blurred.`, + `Blurred image has been uploaded to: gs://${blurredBucketName}/${ + sample.mocks.file.name + }`, ]); }); From d2d045d961ddaa69923bea5f763b9bf42a664e8e Mon Sep 17 00:00:00 2001 From: ace-n Date: Wed, 12 Jun 2019 10:42:57 -0700 Subject: [PATCH 2/3] Fix lint --- functions/imagemagick/test/index.test.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/functions/imagemagick/test/index.test.js b/functions/imagemagick/test/index.test.js index 4377e56bfd..911df164fc 100644 --- a/functions/imagemagick/test/index.test.js +++ b/functions/imagemagick/test/index.test.js @@ -119,7 +119,9 @@ it('blurOffensiveImages blurs unblurred images (Node 6 syntax)', async () => { `The image ${sample.mocks.file.name} has been detected as inappropriate.`, ]); assert.deepStrictEqual(console.log.getCall(2).args, [ - `Image ${sample.mocks.file.name} has been downloaded to /tmp/${sample.mocks.file.name}.`, + `Image ${sample.mocks.file.name} has been downloaded to /tmp/${ + sample.mocks.file.name + }.`, ]); assert.deepStrictEqual(console.log.getCall(3).args, [ `Image ${sample.mocks.file.name} has been blurred.`, @@ -146,7 +148,9 @@ it('blurOffensiveImages blurs unblurred images (Node 8 syntax)', async () => { `The image ${sample.mocks.file.name} has been detected as inappropriate.`, ]); assert.deepStrictEqual(console.log.getCall(2).args, [ - `Image ${sample.mocks.file.name} has been downloaded to /tmp/${sample.mocks.file.name}.`, + `Image ${sample.mocks.file.name} has been downloaded to /tmp/${ + sample.mocks.file.name + }.`, ]); assert.deepStrictEqual(console.log.getCall(3).args, [ `Image ${sample.mocks.file.name} has been blurred.`, From 6d5ac3d66b014b3ac55dfec04a6cb3d8ddc23e9c Mon Sep 17 00:00:00 2001 From: ace-n Date: Wed, 12 Jun 2019 11:09:12 -0700 Subject: [PATCH 3/3] Update prettier version + ACTUALLY fix lint --- functions/imagemagick/index.js | 4 +--- functions/imagemagick/test/index.test.js | 16 ++++------------ package.json | 5 +++-- 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/functions/imagemagick/index.js b/functions/imagemagick/index.js index d8b6658ba1..859e1a550e 100644 --- a/functions/imagemagick/index.js +++ b/functions/imagemagick/index.js @@ -126,9 +126,7 @@ function blurImage(file, blurredBucketName) { }) .then(() => { console.log( - `Blurred image has been uploaded to: gs://${blurredBucketName}/${ - file.name - }` + `Blurred image has been uploaded to: gs://${blurredBucketName}/${file.name}` ); // Delete the temporary file. diff --git a/functions/imagemagick/test/index.test.js b/functions/imagemagick/test/index.test.js index 911df164fc..9d4e7c4501 100644 --- a/functions/imagemagick/test/index.test.js +++ b/functions/imagemagick/test/index.test.js @@ -119,17 +119,13 @@ it('blurOffensiveImages blurs unblurred images (Node 6 syntax)', async () => { `The image ${sample.mocks.file.name} has been detected as inappropriate.`, ]); assert.deepStrictEqual(console.log.getCall(2).args, [ - `Image ${sample.mocks.file.name} has been downloaded to /tmp/${ - sample.mocks.file.name - }.`, + `Image ${sample.mocks.file.name} has been downloaded to /tmp/${sample.mocks.file.name}.`, ]); assert.deepStrictEqual(console.log.getCall(3).args, [ `Image ${sample.mocks.file.name} has been blurred.`, ]); assert.deepStrictEqual(console.log.getCall(4).args, [ - `Blurred image has been uploaded to: gs://${blurredBucketName}/${ - sample.mocks.file.name - }`, + `Blurred image has been uploaded to: gs://${blurredBucketName}/${sample.mocks.file.name}`, ]); }); @@ -148,17 +144,13 @@ it('blurOffensiveImages blurs unblurred images (Node 8 syntax)', async () => { `The image ${sample.mocks.file.name} has been detected as inappropriate.`, ]); assert.deepStrictEqual(console.log.getCall(2).args, [ - `Image ${sample.mocks.file.name} has been downloaded to /tmp/${ - sample.mocks.file.name - }.`, + `Image ${sample.mocks.file.name} has been downloaded to /tmp/${sample.mocks.file.name}.`, ]); assert.deepStrictEqual(console.log.getCall(3).args, [ `Image ${sample.mocks.file.name} has been blurred.`, ]); assert.deepStrictEqual(console.log.getCall(4).args, [ - `Blurred image has been uploaded to: gs://${blurredBucketName}/${ - sample.mocks.file.name - }`, + `Blurred image has been uploaded to: gs://${blurredBucketName}/${sample.mocks.file.name}`, ]); }); diff --git a/package.json b/package.json index 523b727d36..a7a2d54580 100644 --- a/package.json +++ b/package.json @@ -22,8 +22,9 @@ "eslint": "^5.9.0", "eslint-config-prettier": "^4.0.0", "eslint-plugin-node": "^9.0.0", - "eslint-plugin-prettier": "^3.0.0", + "eslint-plugin-prettier": "^3.1.0", "eslint-plugin-promise": "^4.1.1", - "prettier": "^1.15.2" + "prettier": "^1.18.2", + "requestretry": "^4.0.0" } }