Update to OpenPose v1.7.0 Python3 TensorFlow2

refine_video.py

@@ -25,7 +25,7 @@
 import deepdish as dd
 from imageio import imwrite
 
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 from smpl_webuser.serialization import load_model
 
 from src.config import get_config
@@ -36,11 +36,11 @@
 
 
 # Defaults:
-kVidDir = '/home/kanazawa/projects/hmr_sfv/demo_data/videos/'
+kVidDir = 'demo_data/videos/'
 # Where the smoothed results will be stored.
-kOutDir = '/home/kanazawa/projects/hmr_sfv/demo_data/results_smoothed/'
+kOutDir = 'demo_data/results_smoothed/'
 # Holds h5 for each video, which stores OP outputs, after trajectory assignment.
-kOpDir = '/home/kanazawa/projects/hmr_sfv/demo_data/openpose_output'
+kOpDir = 'demo_data/openpose_output'
 
 
 kMaxLength = 1000
@@ -122,7 +122,7 @@ def run_video(frames, per_frame_people, config, out_mov_path):
         result_dict = dd.io.load(out_res_path)
 
     # Render results into video.
-    temp_dir = tempfile.mkdtemp(dir='/mnt/ramdisk/')
+    temp_dir = tempfile.mkdtemp(dir='/tmp/')
     print('writing to %s' % temp_dir)
 
     used_frames = frames[start_fr:end_fr + 1]
@@ -147,7 +147,7 @@ def run_video(frames, per_frame_people, config, out_mov_path):
             'OpenPose Output', (10, 50),
             0,
             1,
-            np.array([0, 0, 0]),
+            (0, 0, 0),
             thickness=3)
         other_vp = np.ones_like(frame)
         other_vp2 = np.ones_like(frame)
@@ -248,7 +248,7 @@ def main(config):
     # Figure out the save name.
     pred_dir = get_pred_prefix(config.load_path)
     # import ipdb; ipdb.set_trace()
-    pred_dir = '/home/kanazawa/projects/hmr_sfv/demo_data/results_smoothed/'
+    pred_dir = './demo_data/results_smoothed/'
 
     for i, vid_path in enumerate(video_paths[:]):
         out_mov_path = join(pred_dir, basename(vid_path).replace('.mp4', '.h5'))
@@ -283,6 +283,7 @@ def main(config):
     # For visualization.
     renderer = SMPLRenderer(img_size=config.img_size, flength=1000.,
                             face_path=config.smpl_face_path)
+    print(config.smpl_model_path)
     smpl = load_model(config.smpl_model_path)
 
     main(config)

run_openpose.py

@@ -15,15 +15,15 @@
 import scipy.signal as signal
 import deepdish as dd
 
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 
 from src.util.renderer import draw_openpose_skeleton
 
-kVidDir = '/home/kanazawa/projects/hmr_sfv/demo_data/videos'
-kOutDir = '/home/kanazawa/projects/hmr_sfv/demo_data/openpose_output'
+kVidDir = './demo_data/videos'
+kOutDir = './demo_data/openpose_output'
 
-kOpenPose = '/scratch1/storage/git_repos/openpose'
-kOpenPoseModel = '/scratch1/storage/git_repos/Realtime_Multi-Person_Pose_Estimation/aj_finetuned_models_170k/'
+kOpenPose = '~/sandbox/openpose'
+kOpenPoseModel = '~/sandbox/models/aj_finetuned_models_170k/'
 
 tf.app.flags.DEFINE_string('video_dir', kVidDir, 'dir of vids')
 tf.app.flags.DEFINE_string('out_dir', kOutDir, 'dir of output')
@@ -73,10 +73,11 @@ def main(unused_argv):
 
     vid_paths = sorted(glob(join(vid_dir, "*.mp4")))
 
-    # cmd_base = '%s/build/examples/openpose/openpose.bin --video %%s --write_keypoint_json %%s --no_display --render_pose 1' % (
+    # cmd_base = '%s/build/examples/openpose/openpose.bin --video %%s --write_json %%s --no_display --render_pose 1' % (
     #     openpose_dir)
     # Maximum accuracy configuration:
-    cmd_base = '%s/build/examples/openpose/openpose.bin --video %%s --write_keypoint_json %%s --net_resolution "1312x736" --scale_number 4 --scale_gap 0.25 --write_images %%s --write_images_format jpg' % (
+    #cmd_base = '%s/build/examples/openpose/openpose.bin --video %%s --write_json %%s --net_resolution "1312x736" --scale_number 4 --scale_gap 0.25 --write_images %%s --write_images_format jpg' % (
+    cmd_base = '%s/build/examples/openpose/openpose.bin --video %%s --write_json %%s --scale_gap 0.25 --write_images %%s --write_images_format jpg' % (
         openpose_dir)
 
     cmd_base += ' --model_folder %s' % FLAGS.op_model_dir    
@@ -172,7 +173,7 @@ def clean_detections(all_kps, vid_path, vis=False):
             end_frame = i
             # Find matching persons.
             iou_scores = []
-            for p_id, p_bboxes in persons.iteritems():
+            for p_id, p_bboxes in iter(persons.items()):
                 last_time, last_bbox, last_kp = p_bboxes[-1]
                 if (i - last_time) > OCCL_THR:
                     ious = -np.ones(len(bboxes))
@@ -235,6 +236,7 @@ def clean_detections(all_kps, vid_path, vis=False):
             plt.pause(1e-3)
 
     # Now clean these people!
+    persons_tbd = [];
     if not vis:
         frames = read_frames(vid_path, 1)
     img_area = frames[0].shape[0] * frames[0].shape[1]
@@ -248,7 +250,8 @@ def clean_detections(all_kps, vid_path, vis=False):
         # print('freq %.2f, score %.2f, size %.2f' % (freq, med_score, median_bbox_area))
         if freq < FREQ_THR:
             print('Rejecting %d bc too suprious: %.2f' % (p_id, freq))
-            del persons[p_id]
+            #del persons[p_id]
+            persons_tbd.append(p_id)
             continue
 
         # if (median_bbox_area) < SIZE_THR:
@@ -258,9 +261,12 @@ def clean_detections(all_kps, vid_path, vis=False):
         #     continue
         if med_score < SCORE_THR:
             print('Rejecting %d bc not confident: %.2f' % (p_id, med_score))
-            del persons[p_id]
+            #del persons[p_id]
+            persons_tbd.append(p_id)
             continue
         print('%d survived with: freq %.2f, score %.2f, size %.2f' % (p_id, freq, med_score, median_bbox_area))
+    for p_id_tbd in persons_tbd:
+        del persons[p_id_tbd]
     print('Total # of ppl trajectories: %d' % len(persons.keys()))
     if len(persons.keys()) == 0:
         return {}
@@ -462,7 +468,7 @@ def read_json(json_path):
         data = json.load(f)
     kps = []
     for people in data['people']:
-        kp = np.array(people['pose_keypoints']).reshape(-1, 3)
+        kp = np.array(people['pose_keypoints_2d']).reshape(-1, 3)
         kps.append(kp)
     return kps
 
@@ -499,10 +505,10 @@ def nonmaxsupp(bboxes0, valid_kps0):
         w = np.maximum(0, xx2 - xx1 + 1)
         h = np.maximum(0, yy2 - yy1 + 1)
         # compute the ratio of overlap
-	overlap = (w * h) / area[idxs[:last]]
+        overlap = (w * h) / area[idxs[:last]]
 
         # delete all indexes from the index list that have
-	idxs = np.delete(idxs, np.concatenate(([last],
+        idxs = np.delete(idxs, np.concatenate(([last],
 			                       np.where(overlap > NMS_THR)[0])))
 
     return bboxes0[pick], valid_kps0[pick]

src/config.py

@@ -28,18 +28,15 @@
 SMPL_FACE_PATH = osp.join(curr_path, 'tf_smpl', 'smpl_faces.npy')
 
 # Default pred-trained model path for the demo.
-PRETRAINED_MODEL = osp.join(model_dir, 'hmr_rotaug_May03_1425/model.ckpt-1111959'
+PRETRAINED_MODEL = osp.join(model_dir, 'hmr_rotaug_May03_1425/model.ckpt-1111959')
 # PRETRAINED_MODEL = osp.join(model_dir, 'model.ckpt-667589')
 
-flags.DEFINE_string('smpl_model_path', SMPL_MODEL_PATH,
-                    'path to the neurtral smpl model')
-flags.DEFINE_string('smpl_face_path', SMPL_FACE_PATH,
-                    'path to smpl mesh faces (for easy rendering)')
+flags.DEFINE_string('smpl_model_path', SMPL_MODEL_PATH, 'path to the neurtral smpl model')
+flags.DEFINE_string('smpl_face_path', SMPL_FACE_PATH, 'path to smpl mesh faces (for easy rendering)')
 flags.DEFINE_string('load_path', PRETRAINED_MODEL, 'path to trained model')
 
 
-flags.DEFINE_integer('img_size', 224,
-                     'Input image size to the network after preprocessing')
+flags.DEFINE_integer('img_size', 224, 'Input image size to the network after preprocessing')
 flags.DEFINE_string('data_format', 'NHWC', 'Data format')
 flags.DEFINE_integer('num_stage', 3, '# of times to iterate regressor')
 flags.DEFINE_boolean('ief', True, 'always true.')

src/models.py

@@ -12,10 +12,10 @@
 from __future__ import division
 from __future__ import print_function
 
-import tensorflow as tf
-import tensorflow.contrib.slim as slim
+import tensorflow.compat.v1 as tf
+import tf_slim as slim
 
-from tensorflow.contrib.layers.python.layers.initializers import variance_scaling_initializer
+from tf_slim.layers.initializers import variance_scaling_initializer
 
 
 def Encoder_resnet(x, is_training=True, weight_decay=0.001, reuse=False):
@@ -33,8 +33,8 @@ def Encoder_resnet(x, is_training=True, weight_decay=0.001, reuse=False):
     - Shape vector: N x 10
     - variables: tf variables
     """
-    from tensorflow.contrib.slim.python.slim.nets import resnet_v2
-    with tf.name_scope("Encoder_resnet", [x]):
+    from tf_slim.nets import resnet_v2
+    with tf.name_scope("Encoder_resnet", values=[x]):
         with slim.arg_scope(
                 resnet_v2.resnet_arg_scope(weight_decay=weight_decay)):
             net, end_points = resnet_v2.resnet_v2_50(
@@ -44,7 +44,7 @@ def Encoder_resnet(x, is_training=True, weight_decay=0.001, reuse=False):
                 reuse=reuse,
                 scope='resnet_v2_50')
             net = tf.squeeze(net, axis=[1, 2])
-    variables = tf.contrib.framework.get_variables('resnet_v2_50')
+    variables = tf.global_variables('resnet_v2_50')
     return net, variables
 
 
@@ -84,7 +84,7 @@ def Encoder_fc3_dropout(x,
             weights_initializer=small_xavier,
             scope='fc3')
 
-    variables = tf.contrib.framework.get_variables(scope)
+    variables = tf.global_variables(scope.name)
     return net, variables
 
 

src/ops.py

@@ -6,7 +6,7 @@
 from __future__ import division
 from __future__ import print_function
 
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 
 
 def keypoint_l1_loss(kp_gt, kp_pred, scale=1., name=None):

src/refiner.py

@@ -16,7 +16,7 @@
 import time
 from os.path import exists
 
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 import numpy as np
 
 
@@ -26,6 +26,9 @@ def __init__(self, config, num_frames, sess=None):
         Args:
           config,,
         """
+        # Disable Eager Execution:
+        tf.compat.v1.disable_eager_execution()
+
         # Config + path
         if not config.load_path:
             raise Exception(
@@ -378,7 +381,7 @@ def predict(self, images, kps, scale_factors, offsets):
         tbegin = time.time()
         num_iter = self.config.num_refine
         loss_records = {}
-        for step in xrange(num_iter):
+        for step in range(num_iter):
             result = self.sess.run(fetch_dict, feed_dict)
             loss_keys = [key for key in all_loss_keys if key in result.keys()]
             total_loss = result['total_loss']

src/tf_smpl/batch_lbs.py

@@ -9,7 +9,7 @@
 from __future__ import division
 from __future__ import print_function
 
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 
 
 def batch_skew(vec, batch_size=None):
@@ -18,7 +18,8 @@ def batch_skew(vec, batch_size=None):
 
     returns N x 3 x 3. Skew_sym version of each matrix.
     """
-    with tf.name_scope("batch_skew", [vec]):
+    print("vec=", vec)
+    with tf.name_scope("batch_skew", values=[vec]):
         if batch_size is None:
             batch_size = vec.shape.as_list()[0]
         col_inds = tf.constant([1, 2, 3, 5, 6, 7])
@@ -167,8 +168,8 @@ def batch_global_rigid_transformation(Rs, Js, parent, rotate_base=False):
       new_J : `Tensor`: N x 24 x 3 location of absolute joints
       A     : `Tensor`: N x 24 4 x 4 relative joint transformations for LBS.
     """
-    with tf.name_scope("batch_forward_kinematics", [Rs, Js]):
-        N = Rs.shape[0].value
+    with tf.name_scope("batch_forward_kinematics", values=[Rs, Js]):
+        N = Rs.shape[0]
         if rotate_base:
             print('Flipping the SMPL coordinate frame!!!!')
             rot_x = tf.constant(

src/tf_smpl/batch_smpl.py

@@ -11,9 +11,9 @@
 from __future__ import print_function
 
 import numpy as np
-import cPickle as pickle
+import pickle
 
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 from .batch_lbs import batch_rodrigues, batch_global_rigid_transformation
 
 
@@ -28,16 +28,16 @@ def __init__(self, pkl_path, joint_type='cocoplus', dtype=tf.float32):
         pkl_path is the path to a SMPL model
         """
         # -- Load SMPL params --
-        with open(pkl_path, 'r') as f:
-            dd = pickle.load(f)    
+        with open(pkl_path, 'rb') as f:
+            dd = pickle.load(f, encoding='iso-8859-1')    
         # Mean template vertices
         self.v_template = tf.Variable(
             undo_chumpy(dd['v_template']),
             name='v_template',
             dtype=dtype,
             trainable=False)
         # Size of mesh [Number of vertices, 3]
-        self.size = [self.v_template.shape[0].value, 3]
+        self.size = [self.v_template.shape[0], 3]
         self.num_betas = dd['shapedirs'].shape[-1]
         # Shape blend shape basis: 6980 x 3 x 10
         # reshaped to 6980*30 x 10, transposed to 10x6980*3
@@ -103,7 +103,7 @@ def __call__(self, beta, theta, get_skin=False, name=None):
         """
 
         with tf.name_scope(name, "smpl_main", [beta, theta]):
-            num_batch = beta.shape[0].value
+            num_batch = beta.shape[0]
 
             # 1. Add shape blend shapes
             # (N x 10) x (10 x 6890*3) = N x 6890 x 3

src/tf_smpl/projection.py

@@ -8,7 +8,7 @@
 from __future__ import division
 from __future__ import print_function
 
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 
 def batch_orth_proj_idrot(X, camera, name=None):
     """