diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 167259647f..c58784b6ff 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -44,6 +44,8 @@ export SENDGRID_API_KEY=$(cat $KOKORO_GFILE_DIR/secrets-sendgrid-api-key.txt) # Configure GCF variables export FUNCTIONS_TOPIC=integration-tests-instance export FUNCTIONS_BUCKET=$GCLOUD_PROJECT + +# functions/speech-to-speech export OUTPUT_BUCKET=$FUNCTIONS_BUCKET # functions/translate @@ -52,6 +54,9 @@ export TRANSLATE_TOPIC=$FUNCTIONS_TOPIC export RESULT_TOPIC=$FUNCTIONS_TOPIC export RESULT_BUCKET=$FUNCTIONS_BUCKET +# functions/imagemagick +export BLURRED_BUCKET_NAME=$GCLOUD_PROJECT-functions + # Configure IoT variables export NODEJS_IOT_EC_PUBLIC_KEY=${KOKORO_GFILE_DIR}/ec_public.pem export NODEJS_IOT_RSA_PRIVATE_KEY=${KOKORO_GFILE_DIR}/rsa_private.pem diff --git a/functions/imagemagick/index.js b/functions/imagemagick/index.js index 859e1a550e..4a64e62667 100644 --- a/functions/imagemagick/index.js +++ b/functions/imagemagick/index.js @@ -18,6 +18,7 @@ // [START functions_imagemagick_setup] const gm = require('gm').subClass({imageMagick: true}); const fs = require('fs'); +const {promisify} = require('util'); const path = require('path'); const {Storage} = require('@google-cloud/storage'); const storage = new Storage(); @@ -30,115 +31,79 @@ const {BLURRED_BUCKET_NAME} = process.env; // [START functions_imagemagick_analyze] // Blurs uploaded images that are flagged as Adult or Violence. -exports.blurOffensiveImages = event => { - const object = event.data || event; // Node 6: event.data === Node 8+: event - - // Exit if this is a deletion or a deploy event. - if (object.resourceState === 'not_exists') { - console.log('This is a deletion event.'); - return; - } else if (!object.name) { - console.log('This is a deploy event.'); - return; - } +exports.blurOffensiveImages = async event => { + const object = event; const file = storage.bucket(object.bucket).file(object.name); const filePath = `gs://${object.bucket}/${object.name}`; - // Ignore already-blurred files (to prevent re-invoking this function) - if (file.name.startsWith('blurred-')) { - console.log(`The image ${file.name} is already blurred.`); - return; - } - console.log(`Analyzing ${file.name}.`); - return client - .safeSearchDetection(filePath) - .catch(err => { - console.error(`Failed to analyze ${file.name}.`, err); - return Promise.reject(err); - }) - .then(([result]) => { - const detections = result.safeSearchAnnotation; - - if ( - detections.adult === 'VERY_LIKELY' || - detections.violence === 'VERY_LIKELY' - ) { - console.log( - `The image ${file.name} has been detected as inappropriate.` - ); - return blurImage(file, BLURRED_BUCKET_NAME); - } else { - console.log(`The image ${file.name} has been detected as OK.`); - } - }); + try { + const [result] = await client.safeSearchDetection(filePath); + const detections = result.safeSearchAnnotation || {}; + + if ( + detections.adult === 'VERY_LIKELY' || + detections.violence === 'VERY_LIKELY' + ) { + console.log(`Detected ${file.name} as inappropriate.`); + return blurImage(file, BLURRED_BUCKET_NAME); + } else { + console.log(`Detected ${file.name} as OK.`); + } + } catch (err) { + console.error(`Failed to analyze ${file.name}.`, err); + return Promise.reject(err); + } }; // [END functions_imagemagick_analyze] // [START functions_imagemagick_blur] // Blurs the given file using ImageMagick, and uploads it to another bucket. -function blurImage(file, blurredBucketName) { +const blurImage = async (file, blurredBucketName) => { const tempLocalPath = `/tmp/${path.parse(file.name).base}`; // Download file from bucket. - return file - .download({destination: tempLocalPath}) - .catch(err => { - console.error('Failed to download file.', err); - return Promise.reject(err); - }) - .then(() => { - console.log( - `Image ${file.name} has been downloaded to ${tempLocalPath}.` - ); - - // Blur the image using ImageMagick. - return new Promise((resolve, reject) => { - gm(tempLocalPath) - .blur(0, 16) - .write(tempLocalPath, (err, stdout) => { - if (err) { - console.error('Failed to blur image.', err); - reject(err); - } else { - resolve(stdout); - } - }); - }); - }) - .then(() => { - console.log(`Image ${file.name} has been blurred.`); - - // Upload result to a different bucket, to avoid re-triggering this function. - // You can also re-upload it to the same bucket + tell your Cloud Function to - // ignore files marked as blurred (e.g. those with a "blurred" prefix) - const blurredBucket = storage.bucket(blurredBucketName); - - // Upload the Blurred image back into the bucket. - return blurredBucket - .upload(tempLocalPath, {destination: file.name}) - .catch(err => { - console.error('Failed to upload blurred image.', err); - return Promise.reject(err); - }); - }) - .then(() => { - console.log( - `Blurred image has been uploaded to: gs://${blurredBucketName}/${file.name}` - ); - - // Delete the temporary file. - return new Promise((resolve, reject) => { - fs.unlink(tempLocalPath, err => { - if (err) { - reject(err); - } else { - resolve(); - } - }); + try { + await file.download({destination: tempLocalPath}); + + console.log(`Downloaded ${file.name} to ${tempLocalPath}.`); + } catch (err) { + console.error('File download failed.', err); + return Promise.reject(err); + } + + await new Promise((resolve, reject) => { + gm(tempLocalPath) + .blur(0, 16) + .write(tempLocalPath, (err, stdout) => { + if (err) { + console.error('Failed to blur image.', err); + reject(err); + } else { + console.log(`Blurred image: ${file.name}`); + resolve(stdout); + } }); - }); -} + }); + + // Upload result to a different bucket, to avoid re-triggering this function. + // You can also re-upload it to the same bucket + tell your Cloud Function to + // ignore files marked as blurred (e.g. those with a "blurred" prefix) + const blurredBucket = storage.bucket(blurredBucketName); + + // Upload the Blurred image back into the bucket. + const gcsPath = `gs://${blurredBucketName}/${file.name}`; + try { + await blurredBucket.upload(tempLocalPath, {destination: file.name}); + console.log(`Uploaded blurred image to: ${gcsPath}`); + } catch (err) { + console.error(`Unable to upload blurred image to ${gcsPath}:`, err); + return Promise.reject(err); + } + + // Delete the temporary file. + return promisify(fs.unlink(tempLocalPath)); +}; // [END functions_imagemagick_blur] diff --git a/functions/imagemagick/package.json b/functions/imagemagick/package.json index 68ec6c7431..7ec9c3b466 100644 --- a/functions/imagemagick/package.json +++ b/functions/imagemagick/package.json @@ -12,7 +12,7 @@ "node": ">=8.0.0" }, "scripts": { - "test": "mocha test/*.test.js --timeout=20000" + "test": "mocha test/*.test.js --timeout=20000 --exit" }, "dependencies": { "@google-cloud/storage": "^3.0.0", @@ -20,13 +20,21 @@ "gm": "^1.23.1" }, "devDependencies": { + "@google-cloud/functions-framework": "^1.1.1", "@google-cloud/nodejs-repo-tools": "^3.3.0", + "child-process-promise": "^2.2.1", "mocha": "^6.0.0", "proxyquire": "^2.1.0", + "request": "^2.88.0", + "requestretry": "^4.0.0", "sinon": "^7.0.0" }, "cloud-repo-tools": { "requiresKeyFile": true, - "requiresProjectId": true + "requiresProjectId": true, + "requiredEnvVars": [ + "FUNCTIONS_BUCKET", + "BLURRED_BUCKET_NAME" + ] } } diff --git a/functions/imagemagick/test/index.test.js b/functions/imagemagick/test/index.test.js index 9d4e7c4501..b4c93a741e 100644 --- a/functions/imagemagick/test/index.test.js +++ b/functions/imagemagick/test/index.test.js @@ -15,169 +15,113 @@ 'use strict'; -const proxyquire = require('proxyquire').noCallThru(); -const sinon = require('sinon'); const assert = require('assert'); const tools = require('@google-cloud/nodejs-repo-tools'); -const vision = require('@google-cloud/vision').v1p1beta1; - -const bucketName = 'my-bucket'; -const blurredBucketName = 'my-blurred-bucket'; -const defaultFileName = 'image.jpg'; - -process.env.BLURRED_BUCKET_NAME = blurredBucketName; - -let VisionStub = sinon.stub(vision, 'ImageAnnotatorClient'); -VisionStub.returns({ - safeSearchDetection: sinon.stub().returns( - Promise.resolve([ - { - safeSearchAnnotation: { - adult: 'VERY_LIKELY', - violence: 'VERY_LIKELY', - }, - }, - ]) - ), +const execPromise = require('child-process-promise').exec; +const path = require('path'); +const {Storage} = require('@google-cloud/storage'); + +const storage = new Storage(); + +let requestRetry = require('requestretry'); +requestRetry = requestRetry.defaults({ + retryStrategy: requestRetry.RetryStrategies.NetworkError, + method: 'POST', + json: true, + retryDelay: 1000, }); -function getSample(filename) { - const file = { - getMetadata: sinon.stub().returns(Promise.resolve([{}])), - setMetadata: sinon.stub().returns(Promise.resolve()), - download: sinon.stub().returns(Promise.resolve()), - bucket: bucketName, - name: filename, - }; - const bucket = { - file: sinon.stub().returns(file), - upload: sinon.stub().returns(Promise.resolve()), - }; - file.bucket = bucket; - const storageMock = { - bucket: sinon.stub().returns(bucket), - }; - const StorageMock = sinon.stub().returns(storageMock); +const BUCKET_NAME = process.env.FUNCTIONS_BUCKET; +const {BLURRED_BUCKET_NAME} = process.env; - const gmMock = sinon.stub().returns({ - blur: sinon.stub().returnsThis(), - write: sinon.stub().yields(), - }); - gmMock.subClass = sinon.stub().returnsThis(); +const safeFileName = 'bicycle.jpg'; +const offensiveFileName = 'zombie.jpg'; - const fsMock = { - unlink: sinon.stub().yields(), - }; +const cwd = path.join(__dirname, '..'); + +const blurredBucket = storage.bucket(BLURRED_BUCKET_NAME); - return { - program: proxyquire('../', { - '@google-cloud/storage': {Storage: StorageMock}, - gm: gmMock, - fs: fsMock, - }), - mocks: { - fs: fsMock, - gm: gmMock, - storage: storageMock, - bucket, - file, - }, +describe('functions/imagemagick tests', () => { + const startFF = port => { + return execPromise( + `functions-framework --target=blurOffensiveImages --signature-type=event --port=${port}`, + {timeout: 15000, shell: true, cwd} + ); }; -} -beforeEach(tools.stubConsole); -afterEach(tools.restoreConsole); + const stopFF = async ffProc => { + try { + return await ffProc; + } catch (err) { + // Timeouts always cause errors on Linux, so catch them + if (err.name && err.name === 'ChildProcessError') { + const {stdout, stderr} = err; + return {stdout, stderr}; + } + + throw err; + } + }; -it('blurOffensiveImages does nothing on delete', async () => { - await getSample(defaultFileName).program.blurOffensiveImages({ - data: {resourceState: 'not_exists'}, - }); - assert.strictEqual(console.log.callCount, 1); - assert.deepStrictEqual(console.log.getCall(0).args, [ - 'This is a deletion event.', - ]); -}); + beforeEach(tools.stubConsole); + afterEach(tools.restoreConsole); -it('blurOffensiveImages does nothing on deploy', async () => { - await getSample(defaultFileName).program.blurOffensiveImages({data: {}}); - assert.strictEqual(console.log.callCount, 1); - assert.deepStrictEqual(console.log.getCall(0).args, [ - 'This is a deploy event.', - ]); -}); + it('blurOffensiveImages detects safe images using Cloud Vision', async () => { + const PORT = 8080; + const ffProc = startFF(PORT); -it('blurOffensiveImages blurs unblurred images (Node 6 syntax)', async () => { - const sample = getSample(defaultFileName); - await sample.program.blurOffensiveImages({ - data: {bucket: bucketName, name: defaultFileName}, - }); - assert.strictEqual(console.log.callCount, 5); - assert.deepStrictEqual(console.log.getCall(0).args, [ - `Analyzing ${sample.mocks.file.name}.`, - ]); - assert.deepStrictEqual(console.log.getCall(1).args, [ - `The image ${sample.mocks.file.name} has been detected as inappropriate.`, - ]); - assert.deepStrictEqual(console.log.getCall(2).args, [ - `Image ${sample.mocks.file.name} has been downloaded to /tmp/${sample.mocks.file.name}.`, - ]); - assert.deepStrictEqual(console.log.getCall(3).args, [ - `Image ${sample.mocks.file.name} has been blurred.`, - ]); - assert.deepStrictEqual(console.log.getCall(4).args, [ - `Blurred image has been uploaded to: gs://${blurredBucketName}/${sample.mocks.file.name}`, - ]); -}); + await requestRetry({ + url: `http://localhost:${PORT}/blurOffensiveImages`, + body: { + data: { + bucket: BUCKET_NAME, + name: safeFileName, + }, + }, + }); -it('blurOffensiveImages blurs unblurred images (Node 8 syntax)', async () => { - const sample = getSample(defaultFileName); + const {stdout} = await stopFF(ffProc); - await sample.program.blurOffensiveImages({ - bucket: bucketName, - name: defaultFileName, + assert.ok(stdout.includes(`Detected ${safeFileName} as OK.`)); }); - assert.strictEqual(console.log.callCount, 5); - assert.deepStrictEqual(console.log.getCall(0).args, [ - `Analyzing ${sample.mocks.file.name}.`, - ]); - assert.deepStrictEqual(console.log.getCall(1).args, [ - `The image ${sample.mocks.file.name} has been detected as inappropriate.`, - ]); - assert.deepStrictEqual(console.log.getCall(2).args, [ - `Image ${sample.mocks.file.name} has been downloaded to /tmp/${sample.mocks.file.name}.`, - ]); - assert.deepStrictEqual(console.log.getCall(3).args, [ - `Image ${sample.mocks.file.name} has been blurred.`, - ]); - assert.deepStrictEqual(console.log.getCall(4).args, [ - `Blurred image has been uploaded to: gs://${blurredBucketName}/${sample.mocks.file.name}`, - ]); -}); -it('blurOffensiveImages ignores safe images', async () => { - VisionStub.restore(); - VisionStub = sinon.stub(vision, 'ImageAnnotatorClient'); - VisionStub.returns({ - safeSearchDetection: sinon.stub().returns( - Promise.resolve([ - { - safeSearchAnnotation: { - adult: 'VERY_UNLIKELY', - violence: 'VERY_UNLIKELY', - }, + it('blurOffensiveImages successfully blurs offensive images', async () => { + const PORT = 8081; + const ffProc = startFF(PORT); + + await requestRetry({ + url: `http://localhost:${PORT}/blurOffensiveImages`, + body: { + data: { + bucket: BUCKET_NAME, + name: offensiveFileName, }, - ]) - ), + }, + }); + + const {stdout} = await stopFF(ffProc); + + assert.ok(stdout.includes(`Blurred image: ${offensiveFileName}`)); + assert.ok( + stdout.includes( + `Uploaded blurred image to: gs://${BLURRED_BUCKET_NAME}/${offensiveFileName}` + ) + ); + + assert.ok( + storage + .bucket(BLURRED_BUCKET_NAME) + .file(offensiveFileName) + .exists(), + 'File uploaded' + ); }); - const sample = getSample(defaultFileName); - await sample.program.blurOffensiveImages({ - data: {bucket: bucketName, name: defaultFileName}, + + after(async () => { + try { + await blurredBucket.file(offensiveFileName).delete(); + } catch (err) { + console.log('Error deleting uploaded file:', err); + } }); - assert.strictEqual(console.log.callCount, 2); - assert.deepStrictEqual(console.log.getCall(0).args, [ - `Analyzing ${sample.mocks.file.name}.`, - ]); - assert.deepStrictEqual(console.log.getCall(1).args, [ - `The image ${sample.mocks.file.name} has been detected as OK.`, - ]); });