diff --git a/doc/jsk_perception/nodes/aws_auto_checkin_app.rst b/doc/jsk_perception/nodes/aws_auto_checkin_app.rst new file mode 100644 index 0000000000..508f226723 --- /dev/null +++ b/doc/jsk_perception/nodes/aws_auto_checkin_app.rst @@ -0,0 +1,83 @@ +aws_auto_cehckin_app.py +========================= + +What is this? +------------- + +.. image:: https://d1.awsstatic.com/Solutions/Solutions%20Category%20Template%20Draft/Solution%20Architecture%20Diagrams/auto-check-in-app-architecture.8baa84b79c2294d035c7b9cee323d7c9ba53a43a.png + +Face recognition using Amazon Rekognition, see +https://aws.amazon.com/solutions/implementations/auto-check-in-app/ +for more info. + +Subscribing Topic +----------------- + + +* ``~image`` (``sensor_msgs/Image``) + + Raw image. + +* ``~face_roi`` (``opencv_apps/FaceArrayStamped``) + + Rectangles on the face of input image. Use ROI value. + ``` + msg.faces[].face.x : X coordinates of the center of the face image in the ~image input + msg.faces[].face.y : Y coordinates of the center of the face image in the ~image input + msg.faces[].face.width : Width of the face image + msg.faces[].face.height : Height of the face iamge + ``` + +Publishing Topic +---------------- + +* ``~face_name`` (``opencv_apps/FaceArrayStamped``) + + Publish recognized face name as well as face image. The face.{x,y,width,height} corresponds to input `face_roi`, that means x, y is the center of face rectangle. + +Parameters +---------- + +* ``~use_window`` (Bool, default: ``False``) + + Show input image on the window, if it is true. + +* ``~env_path`` (String, default: ``env.json``) + + Json file for environment variables to run aws auto-checkin app. You + can find how to generate this file on + https://aws.amazon.com/jp/builders-flash/202004/auto-checkin-app/. + In addition to that, you need to add "UserName" and "UserPassword" + ``` + { + "Region": "%%REGION%%", + "ApiEndpoint" : "%%REST_API_ID%%.execute-api.%%REGION%%.amazonaws.com/prod/rekognize_face", + "CognitoUserPoolId": "%%COGNITO_USER_POOL_ID%%", + "CognitoUserPoolClientId": "%%COGNITO_USER_POOL_CLIENT_ID%%", + "FaceAreaThreshold": 1e4, + "FaceMarginRatio": 0.2, + "FaceSimilarityThreshold": 90, + "CroppedImageWidth": 540, + "CroppedImageHeight": 540, + "NameTtlSec": 10, + "UseDeepLeaningForDetector": true, + "UserName": "%%YOUR_USER_NAME%%", + "UserPassword": "%%YOUR_PASSWORD%%" + } + ``` + +Sample +------ + +.. code-block:: bash + + roslaunch jsk_perception sample_aws_auto_checkin_app.launch use_window:=true + + +For JSK user, Download `env.json` file from +[Gdrive](https://drive.google.com/file/d/1Wl-yzRD8LNipqcE4jQfOOcJQZJZbdcMW/view?usp=sharing) +and put this under `/tmp` directory to run sample code. + +To add new people to face database, add face image file to [Amazon +S3](https://console.aws.amazon.com/s3), +`auto-check-in-gapp-register...` buckets diff --git a/jsk_perception/node_scripts/aws_auto_checkin_app.py b/jsk_perception/node_scripts/aws_auto_checkin_app.py new file mode 100755 index 0000000000..08b594b8a0 --- /dev/null +++ b/jsk_perception/node_scripts/aws_auto_checkin_app.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python + +## +## Modified from awslabs/auto-check-in-app +## +## https://github.com/awslabs/auto-check-in-app/blob/master/source/frontend/controller.py +## +## +## Author: Yuli-disney +## Kei Okada + +import rospy + +import message_filters +from sensor_msgs.msg import CompressedImage +from opencv_apps.msg import FaceArrayStamped, Face, Rect + +import numpy as np + +import boto3 +import cv2 +import datetime +import json +import requests +import math + +class AutoCheckIn(object): + + def _get_id_token_by_cognito(self, username, password): + client = boto3.client('cognito-idp', self.REGION) + rospy.loginfo("Initiate User Auth for {}".format(username)) + response = client.initiate_auth( + ClientId=self.COGNITO_USERPOOL_CLIENT_ID, + AuthFlow='USER_PASSWORD_AUTH', + AuthParameters={ + 'USERNAME': username, + 'PASSWORD': password + } + ) + return response['AuthenticationResult']['IdToken'] + + def __init__(self): + rospy.init_node('aws_auto_checkin_service') + rospy.loginfo("ROS node initialized as {}".format(rospy.get_name())) + + env_path = rospy.get_param('~env_path', 'env.json') + rospy.loginfo("Loading AutoCheckin env variables from {}".format(env_path)) + try: + with open(env_path) as env_json: + env = json.load(env_json) + except IOError: + rospy.logerr('Cannot open "{}".\nCopy "default.env.json" file as a new file called "env.json" and edit parameters in it.'.format(env_path)) + raise + + try: + self.API_ENDPOINT = env['ApiEndpoint'] + self.FACE_AREA_THRESHOLD = env['FaceAreaThreshold'] + self.NAME_TTL_SEC = env['NameTtlSec'] + self.FACE_SIMILARITY_THRESHOLD = env['FaceSimilarityThreshold'] + self.COGNITO_USERPOOL_ID = env['CognitoUserPoolId'] + self.COGNITO_USERPOOL_CLIENT_ID = env['CognitoUserPoolClientId'] + self.REGION = env['Region'] + except KeyError: + print('Invalid config file') + raise + + self.id_token = self._get_id_token_by_cognito(env['UserName'], env['UserPassword']) + + self.use_window = rospy.get_param('~use_window', False) + rospy.loginfo("Launch image window : {}".format(self.use_window)) + + self.name_pub = rospy.Publisher('face_name', FaceArrayStamped, queue_size=1) + self.image_sub = message_filters.Subscriber('{}/compressed'.format(rospy.resolve_name('image')), CompressedImage) + # we wan to use RegionOfInterest, but it message_filters requires + # header information, so use CameraInfo + self.roi_sub = message_filters.Subscriber('face_roi', FaceArrayStamped) + self.ts = message_filters.ApproximateTimeSynchronizer([self.image_sub, self.roi_sub], 10, 1, allow_headerless = True) + self.ts.registerCallback(self.callback) + rospy.loginfo("Waiting for {} and {}".format(self.image_sub.name, self.roi_sub.name)) + + def findface(self, face_image): + area = face_image.shape[0] * face_image.shape[1] + if area < self.FACE_AREA_THRESHOLD / 10: + return None + if area > self.FACE_AREA_THRESHOLD * 2: + # resize + ratio = math.sqrt(area / (self.FACE_AREA_THRESHOLD * 2)) + face_image = cv2.resize(face_image, (int( + face_image.shape[1] / ratio), int(face_image.shape[0] / ratio))) + + _, encoded_face_image = cv2.imencode('.jpg', face_image) + + # Call API + try: + endpoint = 'https://' + self.API_ENDPOINT + t = datetime.datetime.utcnow() + amz_date = t.strftime('%Y%m%dT%H%M%SZ') + headers = { + 'Content-Type': 'image/jpg', + 'X-Amz-Date':amz_date, + 'Authorization': self.id_token + } + request_parameters = encoded_face_image.tostring() + res = requests.post(endpoint, data=request_parameters, headers=headers).json() + rospy.loginfo("responce : {}".format(res)) + # renponse samples: + # {'result': 'OK', 'name': 'hoge', 'similarity': 95.15} + # {'result': 'NO_MATCH', 'name': '', 'similarity': 0} + # {'result': 'INVALID', 'name': '', 'similarity': 0} + + result = res['result'] + except Exception as e: + print(e) + + else: + if result == 'OK': + name = res['name'] + similarity = res['similarity'] + if similarity > self.FACE_SIMILARITY_THRESHOLD: + return res + + return None + + def callback(self, image, roi): + # decode compressed image + np_arr = np.fromstring(image.data, np.uint8) + img = cv2.imdecode(np_arr, cv2.IMREAD_COLOR) + if image.format != "rgb8; jpeg compressed bgr8": + img = img[:, :, ::-1] + + if self.use_window: + img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) + img_gray = cv2.cvtColor(img_gray, cv2.COLOR_GRAY2BGR) + + faces = FaceArrayStamped() + faces.header = image.header + faces.faces = [] + for face in roi.faces: + try: + cx = int(face.face.x) + cy = int(face.face.y) + w = int(face.face.width) + h = int(face.face.height) + except Exception as e: + rospy.logerr(e) + return + + ret = self.findface(img[cy-h/2:cy+h/2,cx-w/2:cx+w/2]) + if ret != None: + faces.faces.append(Face(face=Rect(cx, cy, w, h), + label=ret['name'], + confidence=ret['similarity'])) + + if self.use_window: # copy colored face rectangle to img_gray + img_gray[cy-h/2:cy+h/2,cx-h/2:cx+w/2] = img[cy-h/2:cy+h/2,cx-w/2:cx+w/2] + + self.name_pub.publish(faces) + + if self.use_window: + cv2.imshow(image._connection_header['topic'], img_gray) + cv2.waitKey(1) + + +if __name__ == '__main__': + auto = AutoCheckIn() + rospy.spin() + diff --git a/jsk_perception/sample/sample_aws_auto_checkin_app.launch b/jsk_perception/sample/sample_aws_auto_checkin_app.launch new file mode 100644 index 0000000000..09ec603f88 --- /dev/null +++ b/jsk_perception/sample/sample_aws_auto_checkin_app.launch @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + use_window: $(arg use_window) + env_path: /tmp/env.json + + + +