Sending Camera Snapshots to Amazon Rekognition

This is a quick little flow I pulled together to PoC the usage of Amazon's Rekognition to detect content in my camera snapshots. I have it setup to grab the snapshot when motion is detected, look for objects/labels in the picture, draw a box around them, and then upload the final image to google photos.

For this I used a couple of different nodes and needed to create an account on Amazon AWS to obtain access to Rekognition, and create google photo's account to upload the image to.

• npm i node-red-contrib-image-output - This takes an image passed as a payload and displays it in the workspace.
• npm i node-red-contrib-aws - This is the Amazon AWS SDK.
• npm i node-red-contrib-post-object-detection - This includes the node 'bbox-image', which draws boxes on images. A couple of notes with this package. 1 - It has a second node which is dependant on Tensor Flow and will raise an error during install. It is not needed for my use case. 2 - I needed to install these packages to get node canvas to install sudo apt-get install build-essential libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev.
• npm i NorthernMan54/node-red-contrib-easybotics-gphoto#patch-2 - This is a patched version of the gphoto-uploader that sends a message on upload.
• npm i node-red-contrib-image-tools

You will need to create an Amazon AWS Account to use Rekognition, details are here.

And once you have created the account, you will need to update the AWS Config node with your region, Access ID and Secret Key. And the Upload Photo with your gphoto account.

Also the flow uses your Desktop directory as a temporary file location prior to uploading to ghoto. The photo file name and path can be changed in the Date/Time Formatter node.

[{"id":"a031a6c2.0d6e4","type":"tab","label":"Flow 1","disabled":false,"info":""},{"id":"577815d9.6fd11c","type":"hb-control","z":"a031a6c2.0d6e4","name":"Eufy Porch - Dummy","Homebridge":"Indoor Cam 2K","Manufacturer":"Eufy","Service":"Camera Control","device":"Indoor Cam 2K65:7F:96:9B:5F:C2EufyEufy Porch - Dummy00000111","conf":"6ec121fe.6f7bb","outputs":1,"x":460,"y":60,"wires":[["80b32353.53112","2ba600b6.b7f1e8"]]},{"id":"29c02f88.c9272","type":"debug","z":"a031a6c2.0d6e4","name":"","active":true,"tosidebar":true,"console":true,"tostatus":true,"complete":"payload","targetType":"msg","x":220,"y":520,"wires":[]},{"id":"e568ed2b.70a048","type":"inject","z":"a031a6c2.0d6e4","name":"Test Button","topic":"","payload":"","payloadType":"date","repeat":"","crontab":"","once":false,"onceDelay":0.1,"x":170,"y":120,"wires":[["577815d9.6fd11c"]]},{"id":"bcf6eccf.bf33c","type":"AWS Rekognition","z":"a031a6c2.0d6e4","aws":"1df182d1.56bc4d","operation":"DetectText","SourceImage":"","TargetImage":"","CollectionId":"","Input":"","Output":"","Name":"","Settings":"","RoleArn":"","FaceIds":"","Image":"","Id":"","JobId":"","FaceId":"","Video":"","name":"","x":430,"y":180,"wires":[[],[]]},{"id":"80b32353.53112","type":"image","z":"a031a6c2.0d6e4","name":"","width":160,"data":"payload","dataType":"msg","thumbnail":false,"active":true,"pass":false,"outputs":0,"x":760,"y":60,"wires":[]},{"id":"2ba600b6.b7f1e8","type":"function","z":"a031a6c2.0d6e4","name":"Convert Image","func":"msg.Image={Bytes:msg.payload};\nmsg.MinConfidence = 90;\nreturn msg;","outputs":1,"noerr":0,"x":160,"y":240,"wires":[["bcf6eccf.bf33c","e82a7359.5bf87","f4592d1c.6462d","7c93d533.4b075c"]]},{"id":"f4592d1c.6462d","type":"AWS Rekognition","z":"a031a6c2.0d6e4","aws":"1df182d1.56bc4d","operation":"DetectLabels","SourceImage":"","TargetImage":"","CollectionId":"","Input":"","Output":"","Name":"","Settings":"","RoleArn":"","FaceIds":"","Image":"","Id":"","JobId":"","FaceId":"","Video":"","name":"","x":430,"y":300,"wires":[["2cbfc01d.f481","17f92c78.6588d4"],[]]},{"id":"2cbfc01d.f481","type":"debug","z":"a031a6c2.0d6e4","name":"","active":true,"tosidebar":true,"console":true,"tostatus":true,"complete":"payload","targetType":"msg","x":680,"y":220,"wires":[]},{"id":"e82a7359.5bf87","type":"AWS Rekognition","z":"a031a6c2.0d6e4","aws":"1df182d1.56bc4d","operation":"DetectFaces","SourceImage":"","TargetImage":"","CollectionId":"","Input":"","Output":"","Name":"","Settings":"","RoleArn":"","FaceIds":"","Image":"","Id":"","JobId":"","FaceId":"","Video":"","name":"","x":430,"y":240,"wires":[[],[]]},{"id":"7106b589.34da44","type":"debug","z":"a031a6c2.0d6e4","name":"","active":true,"tosidebar":true,"console":true,"tostatus":true,"complete":"true","targetType":"full","x":570,"y":540,"wires":[]},{"id":"3843ca4d.3542ee","type":"hb-event","z":"a031a6c2.0d6e4","name":"Porch Motion","Homebridge":"homebridge","Manufacturer":"Tasmota","Service":"Motion Sensor","device":"homebridgeCC:22:3D:E3:CF:34TasmotaPorch Motion00000085","conf":"6ec121fe.6f7bb","x":170,"y":60,"wires":[["577815d9.6fd11c"]]},{"id":"906ce7e8.d3f178","type":"bbox-image","z":"a031a6c2.0d6e4","strokeWidth":"2","fontSize":"10","objectsProp":"objects","objectsPropType":"msgPayload","imageProp":"image","imagePropType":"msgPayload","name":"","x":390,"y":400,"wires":[["702830db.a11d78","4882845d.79f884"]]},{"id":"702830db.a11d78","type":"image","z":"a031a6c2.0d6e4","name":"","width":160,"data":"payload","dataType":"msg","thumbnail":false,"active":true,"pass":false,"outputs":0,"x":820,"y":460,"wires":[]},{"id":"7fc9aa03.d3aa54","type":"join","z":"a031a6c2.0d6e4","name":"","mode":"custom","build":"merged","property":"payload","propertyType":"msg","key":"topic","joiner":"\\n","joinerType":"str","accumulate":true,"timeout":"","count":"","reduceRight":false,"reduceExp":"","reduceInit":"","reduceInitType":"","reduceFixup":"","x":190,"y":400,"wires":[["906ce7e8.d3f178","29c02f88.c9272"]]},{"id":"7c93d533.4b075c","type":"function","z":"a031a6c2.0d6e4","name":"Convert Image","func":"msg.payload = {};\nmsg.payload.image = msg.Image.Bytes;\ndelete msg.Image;\nreturn msg;\n","outputs":1,"noerr":0,"x":160,"y":320,"wires":[["7fc9aa03.d3aa54"]]},{"id":"fbaf1c4e.e97c1","type":"file","z":"a031a6c2.0d6e4","name":"","filename":"","appendNewline":true,"createDir":false,"overwriteFile":"true","encoding":"none","x":810,"y":400,"wires":[["3f3f0309.8b1124"]]},{"id":"17f92c78.6588d4","type":"function","z":"a031a6c2.0d6e4","name":"BBox","func":"msg.payload.objects = [];\nmsg.payload.Labels.forEach(function(label) {\n if (label.Instances.length > 0 )\n {\n label.Instances.forEach(function(instance){\n msg.payload.objects.push({ bbox: [instance.BoundingBox.Left, instance.BoundingBox.Top, instance.BoundingBox.Width, instance.BoundingBox.Height],\n className: label.Name,\n score: instance.Confidence});\n })\n }\n})\ndelete msg.payload.Labels;\nmsg.complete = true;\nreturn msg;\n/*\n{\"Labels\":[{\"Name\":\"Path\",\"Confidence\":98.61656951904297,\"Instances\":[],\"Parents\":[]},\n{\"Name\":\"Road\",\"Confidence\":96.95530700683594,\"Instances\":[],\"Parents\":[]},\n{\"Name\":\"Furniture\",\"Confidence\":96.45349884033203,\"Instances\":[],\"Parents\":[]},\n{\"Name\":\"Chair\",\"Confidence\":96.45349884033203,\"Instances\":[{\"BoundingBox\":{\"Width\":0.07030118256807327,\"Height\":0.16095371544361115,\"Left\":0.7400304675102234,\"Top\":0.17652830481529236},\"Confidence\":96.45349884033203},{\"BoundingBox\":{\"Width\":0.1145857498049736,\"Height\":0.24060086905956268,\"Left\":0.7839818596839905,\"Top\":0.2865181267261505},\"Confidence\":83.25431060791016}],\"Parents\":[{\"Name\":\"Furniture\"}]},\n{\"Name\":\"Walkway\",\"Confidence\":93.5756607055664,\"Instances\":[],\"Parents\":[{\"Name\":\"Path\"}]},\n{\"Name\":\"Dirt Road\",\"Confidence\":91.7917709350586,\"Instances\":[],\"Parents\":[{\"Name\":\"Road\"}]},\n{\"Name\":\"Gravel\",\"Confidence\":91.7917709350586,\"Instances\":[],\"Parents\":[{\"Name\":\"Road\"}]}],\n\"LabelModelVersion\":\"2.0\"}\n\n{\n bbox: [x, y, w, h],\n className: string,\n score: number\n}\n*/\n","outputs":1,"noerr":0,"x":650,"y":300,"wires":[["7fc9aa03.d3aa54"]]},{"id":"4882845d.79f884","type":"moment","z":"a031a6c2.0d6e4","name":"","topic":"","input":"","inputType":"date","inTz":"America/Toronto","adjAmount":0,"adjType":"days","adjDir":"add","format":"[Desktop/Cottage_Porch_]HH_mm_ss_MMM_D_YYYY[.jpeg]","locale":"en-CA","output":"filename","outputType":"msg","outTz":"America/Toronto","x":620,"y":400,"wires":[["fbaf1c4e.e97c1"]]},{"id":"84c2c346.d2ac48","type":"upload-photo","z":"a031a6c2.0d6e4","name":"","fileName":"","albumName":"CottageRekognition","account":"6777ea6b.c99474","x":600,"y":460,"wires":[["7106b589.34da44","8d1a76e5.8299f"]]},{"id":"cb949dc9.3969b8","type":"debug","z":"a031a6c2.0d6e4","name":"","active":true,"tosidebar":true,"console":true,"tostatus":true,"complete":"true","targetType":"full","x":530,"y":640,"wires":[]},{"id":"3f3f0309.8b1124","type":"function","z":"a031a6c2.0d6e4","name":"Filename","func":"msg.payload = {};\nmsg.payload.fileName = msg.filename;\ndelete msg.Image;\nreturn msg;\n","outputs":1,"noerr":0,"x":420,"y":460,"wires":[["84c2c346.d2ac48","cb949dc9.3969b8"]]},{"id":"8d1a76e5.8299f","type":"file","z":"a031a6c2.0d6e4","name":"Delete","filename":"","appendNewline":true,"createDir":false,"overwriteFile":"delete","encoding":"none","x":760,"y":600,"wires":[[]]},{"id":"6ec121fe.6f7bb","type":"hb-conf","z":"","username":"031-45-154"},{"id":"1df182d1.56bc4d","type":"amazon config","z":"","name":"AWS","region":"us-east-1","proxyRequired":false,"proxy":""},{"id":"6777ea6b.c99474","type":"photo-account","z":""}]

WIP Version 2 utilizing Motion Eye as the Server

[{"id":"874a9df7.90b8e","type":"inject","z":"d5e30e05.a3ab","name":"Test Button","topic":"","payload":"{\"hour\":\"20\",\"frame\":\"01\",\"upload\":\"yes\",\"month\":\"11\",\"second\":\"41\",\"minute\":\"34\",\"filename\":\"/var/lib/motioneye/Camera1/2020-11-26/12-25-17.jpg\",\"year\":\"2020\",\"event\":\"01\",\"day\":\"25\"}","payloadType":"json","repeat":"","crontab":"","once":false,"onceDelay":0.1,"x":110,"y":100,"wires":[["8b5899fb.7f5a2"]]},{"id":"5d96d73d.acc3d","type":"AWS Rekognition","z":"d5e30e05.a3ab","aws":"4430f76f.5cf7b8","operation":"DetectText","SourceImage":"","TargetImage":"","CollectionId":"","Input":"","Output":"","Name":"","Settings":"","RoleArn":"","FaceIds":"","Image":"","Id":"","JobId":"","FaceId":"","Video":"","name":"","x":410,"y":180,"wires":[[],[]]},{"id":"db1cb325.dda74","type":"image","z":"d5e30e05.a3ab","name":"","width":160,"data":"payload","dataType":"msg","thumbnail":false,"active":true,"pass":false,"outputs":0,"x":700,"y":100,"wires":[]},{"id":"5b1b4a05.4cc924","type":"function","z":"d5e30e05.a3ab","name":"Convert Image","func":"msg.Image={Bytes:msg.payload};\nmsg.MinConfidence = 90;\nreturn msg;","outputs":1,"noerr":0,"x":140,"y":180,"wires":[["e7d5b990.b0896"]]},{"id":"e7d5b990.b0896","type":"AWS Rekognition","z":"d5e30e05.a3ab","aws":"4430f76f.5cf7b8","operation":"DetectLabels","SourceImage":"","TargetImage":"","CollectionId":"","Input":"","Output":"","Name":"","Settings":"","RoleArn":"","FaceIds":"","Image":"","Id":"","JobId":"","FaceId":"","Video":"","name":"","x":410,"y":260,"wires":[["9009b0b8.09e8a8","6695cc8c.48d9e4"],[]]},{"id":"9009b0b8.09e8a8","type":"debug","z":"d5e30e05.a3ab","name":"","active":true,"tosidebar":true,"console":false,"tostatus":true,"complete":"true","targetType":"full","x":670,"y":280,"wires":[]},{"id":"da824754.50fc8","type":"AWS Rekognition","z":"d5e30e05.a3ab","aws":"4430f76f.5cf7b8","operation":"DetectFaces","SourceImage":"","TargetImage":"","CollectionId":"","Input":"","Output":"","Name":"","Settings":"","RoleArn":"","FaceIds":"","Image":"","Id":"","JobId":"","FaceId":"","Video":"","name":"","x":410,"y":220,"wires":[[],[]]},{"id":"d8eaea13.41bd4","type":"bbox-image","z":"d5e30e05.a3ab","strokeWidth":"2","fontSize":"10","objectsProp":"objects","objectsPropType":"msgPayload","imageProp":"image","imagePropType":"msgPayload","name":"","x":450,"y":320,"wires":[["549c8838.e66b8","f74db9cf.7d1fd"]]},{"id":"549c8838.e66b8","type":"image","z":"d5e30e05.a3ab","name":"","width":160,"data":"payload","dataType":"msg","thumbnail":false,"active":true,"pass":false,"outputs":0,"x":700,"y":340,"wires":[]},{"id":"938fc19c.4fcc3","type":"upload-photo","z":"d5e30e05.a3ab","name":"","fileName":"","albumName":"EyeRekognition","account":"7db8b07.486c9d","x":340,"y":400,"wires":[["3482be4e.6ec222"]]},{"id":"46cd1437.0bd7b4","type":"http in","z":"d5e30e05.a3ab","name":"Driveway","url":"/Driveway","method":"post","upload":false,"swaggerDoc":"","x":100,"y":40,"wires":[["63d966ce.ebdf2","8b5899fb.7f5a2","db7b8ef7.5ad598"]]},{"id":"63d966ce.ebdf2","type":"http response","z":"d5e30e05.a3ab","name":"","statusCode":"200","headers":{},"x":280,"y":40,"wires":[]},{"id":"8fb15ce4.67a29","type":"http request","z":"d5e30e05.a3ab","name":"","method":"GET","ret":"bin","paytoqs":false,"url":"","tls":"","persist":false,"proxy":"","authType":"","x":470,"y":100,"wires":[["db1cb325.dda74","5b1b4a05.4cc924","502895c9.722e7c"]]},{"id":"8b5899fb.7f5a2","type":"function","z":"d5e30e05.a3ab","name":"URL","func":"msg.url=\"http://hank.local:8765/picture/1/download/\" + msg.payload.filename;\ndelete msg.payload;\nif (!msg.url.endsWith(\".mp4\"))\nreturn msg;","outputs":1,"noerr":0,"x":270,"y":100,"wires":[["8fb15ce4.67a29"]]},{"id":"502895c9.722e7c","type":"debug","z":"d5e30e05.a3ab","name":"","active":true,"tosidebar":true,"console":false,"tostatus":true,"complete":"true","targetType":"full","x":670,"y":240,"wires":[]},{"id":"526a7928.9ff458","type":"inject","z":"d5e30e05.a3ab","name":"Movie Button","topic":"","payload":"{\"hour\":\"20\",\"frame\":\"01\",\"upload\":\"yes\",\"month\":\"11\",\"second\":\"41\",\"minute\":\"34\",\"filename\":\"/var/lib/motioneye/Camera1/2020-11-25/21-50-53.mp4\",\"year\":\"2020\",\"event\":\"01\",\"day\":\"25\"}","payloadType":"json","repeat":"","crontab":"","once":false,"onceDelay":0.1,"x":110,"y":140,"wires":[["8b5899fb.7f5a2"]]},{"id":"6695cc8c.48d9e4","type":"function","z":"d5e30e05.a3ab","name":"BBox","func":"msg.payload.objects = [];\nmsg.labels = '';\nmsg.payload.Labels.forEach(function(label) {\n    msg.labels = msg.labels + '\\n' + label.Name;\n    if (label.Instances.length > 0 )\n        {\n            label.Instances.forEach(function(instance){\n                msg.payload.objects.push({  bbox: [instance.BoundingBox.Left, instance.BoundingBox.Top, instance.BoundingBox.Width, instance.BoundingBox.Height],\n  className: label.Name,\n  score: instance.Confidence});\n            })\n        }\n})\ndelete msg.payload.Labels;\nmsg.payload.image = msg.Image.Bytes;\ndelete msg.Image;\nmsg.payload.fileName = msg.payload.url;\nreturn msg;\n/*\n{\"Labels\":[{\"Name\":\"Path\",\"Confidence\":98.61656951904297,\"Instances\":[],\"Parents\":[]},\n{\"Name\":\"Road\",\"Confidence\":96.95530700683594,\"Instances\":[],\"Parents\":[]},\n{\"Name\":\"Furniture\",\"Confidence\":96.45349884033203,\"Instances\":[],\"Parents\":[]},\n{\"Name\":\"Chair\",\"Confidence\":96.45349884033203,\"Instances\":[{\"BoundingBox\":{\"Width\":0.07030118256807327,\"Height\":0.16095371544361115,\"Left\":0.7400304675102234,\"Top\":0.17652830481529236},\"Confidence\":96.45349884033203},{\"BoundingBox\":{\"Width\":0.1145857498049736,\"Height\":0.24060086905956268,\"Left\":0.7839818596839905,\"Top\":0.2865181267261505},\"Confidence\":83.25431060791016}],\"Parents\":[{\"Name\":\"Furniture\"}]},\n{\"Name\":\"Walkway\",\"Confidence\":93.5756607055664,\"Instances\":[],\"Parents\":[{\"Name\":\"Path\"}]},\n{\"Name\":\"Dirt Road\",\"Confidence\":91.7917709350586,\"Instances\":[],\"Parents\":[{\"Name\":\"Road\"}]},\n{\"Name\":\"Gravel\",\"Confidence\":91.7917709350586,\"Instances\":[],\"Parents\":[{\"Name\":\"Road\"}]}],\n\"LabelModelVersion\":\"2.0\"}\n\n{\n  bbox: [x, y, w, h],\n  className: string,\n  score: number\n}\n*/\n","outputs":1,"noerr":0,"x":130,"y":320,"wires":[["8ff80d7b.a38cb"]]},{"id":"8ff80d7b.a38cb","type":"jimp-image","z":"d5e30e05.a3ab","name":"","data":"payload.image","dataType":"msg","ret":"buf","parameter1":"FONT_SANS_32_WHITE","parameter1Type":"jimpFont","parameter2":"100","parameter2Type":"num","parameter3":"50","parameter3Type":"num","parameter4":"labels","parameter4Type":"msg","parameter5":"","parameter5Type":"msg","parameter6":"","parameter6Type":"msg","parameter7":"","parameter7Type":"msg","parameter8":"","parameter8Type":"msg","sendProperty":"payload.image","sendPropertyType":"msg","parameterCount":6,"jimpFunction":"print","selectedJimpFunction":{"name":"print","fn":"print","description":"Print text to the image","parameters":[{"name":"font|str","type":"jimpFont","required":true,"hint":"font to print. NOTE: This can be one of the presets or the path to a fnt file"},{"name":"x","type":"num","required":true,"hint":"x coordinate to print text"},{"name":"y","type":"num","required":true,"hint":"y coordinate to print text"},{"name":"text","type":"str","required":true,"hint":"text to print"},{"name":"maxWidth","type":"num","required":false,"hint":"wrap text at maxWidth"},{"name":"maxHeight","type":"num","required":false,"hint":"max height"}]},"x":290,"y":320,"wires":[["d8eaea13.41bd4"]]},{"id":"f74db9cf.7d1fd","type":"function","z":"d5e30e05.a3ab","name":"gPhoto","func":"var payload = {};\npayload.image = msg.payload;\npayload.fileName = msg.url;\nmsg.payload = payload;\nreturn(msg);\n","outputs":1,"noerr":0,"x":140,"y":400,"wires":[["938fc19c.4fcc3"]]},{"id":"3482be4e.6ec222","type":"debug","z":"d5e30e05.a3ab","name":"","active":true,"tosidebar":true,"console":false,"tostatus":true,"complete":"true","targetType":"full","x":510,"y":400,"wires":[]},{"id":"db7b8ef7.5ad598","type":"debug","z":"d5e30e05.a3ab","name":"","active":true,"tosidebar":true,"console":false,"tostatus":true,"complete":"true","targetType":"full","x":290,"y":580,"wires":[]},{"id":"4430f76f.5cf7b8","type":"amazon config","z":"996e60b7.f4d13","name":"AWS","region":"us-east-1","proxyRequired":false,"proxy":""},{"id":"7db8b07.486c9d","type":"photo-account","z":""}]