implement custom interpolators for labelbox

deeplake/integrations/labelbox/labelbox_converter.py

@@ -16,6 +16,7 @@ def __init__(
         self.regsistered_actions = dict()
         self.label_mappings = dict()
         self.values_cache = dict()
+        self.registered_interpolators = dict()
 
         self.project = project
         self.project_id = project_id
@@ -194,29 +195,49 @@ def find_object_with_feature_id_(self, frame, feature_id):
             return frame
 
         return None
+
+    def existing_sub_ranges_(self, frames, range):
+        sub_ranges = [(range[0], range[1])]
+        for i in range[0], range[1]:
+            if str(i) in frames:
+                continue
+            sub_ranges[-1] = (sub_ranges[-1][0], i)
+            sub_ranges.append((i, range[1]))
+        return sub_ranges
+
 
     def parse_segments_(self, segments, frames, offset):
         print('total segments count to parse:', len(segments))
         for feature_id, ranges in segments.items():
             print('parsing segments with feature id: ', feature_id)
             for r in tqdm.tqdm(ranges):
-                assert str(r[0]) in frames
-                obj = self.find_object_with_feature_id_(frames[str(r[0])], feature_id)
-                assert obj is not None
-                for i in range(r[0] + 1, r[1]):
-                    if str(i) in frames:
-                        new_obj = self.find_object_with_feature_id_(
-                            frames[str(i)], feature_id
-                        )
-                    else:
-                        new_obj = None
-                    if new_obj:
-                        obj = new_obj
-                        # no need to update the frame if the object is present in the frame
+                sub_ranges = self.existing_sub_ranges_(frames, r)
+                for st, en in sub_ranges:
+                    assert str(st) in frames
+                    assert str(en) in frames
+
+                    start = self.find_object_with_feature_id_(frames[str(st)], feature_id)
+                    end = self.find_object_with_feature_id_(frames[str(en)], feature_id)
+
+                    assert start
+                    assert end
+
+                    if start == end:
                         continue
-                    self.regsistered_actions[obj["feature_schema_id"]](
-                        offset + i - 1, obj
-                    )
+
+                    assert start["feature_schema_id"] == end["feature_schema_id"]
+
+                    for i in range(st + 1, en):
+                        if str(i) in frames:
+                            obj = self.find_object_with_feature_id_(frames[str(i)], feature_id)
+                        else:
+                            if st['feature_schema_id'] in self.registered_interpolators:
+                                obj = self.registered_interpolators[start["feature_schema_id"]](start, end, (i - st) / (en - st))
+                            else:
+                                obj = end
+
+                            self.regsistered_actions[obj["feature_schema_id"]](offset + i - 1, obj)
+
 
     def apply_cached_values_(self, cache):
         print('applying cached values')

deeplake/integrations/labelbox/labelbox_utils.py

@@ -153,7 +153,8 @@ def labelbox_get_project_json_with_id_(client, project_id, fail_on_error=False):
         "project_details": True,
         "label_details": False,
         "performance_details": False,
-        "interpolated_frames": True,
+        # interpolated_frames does not work with the latest version of the API 6.2.0
+        "interpolated_frames": False,
         "embeddings": False,
     }
 

deeplake/integrations/labelbox/v3_converters.py

@@ -1,6 +1,7 @@
 from PIL import Image
 import urllib.request
 import numpy as np
+import copy
 
 
 def bbox_converter_(obj, converter, tensor_name, context, generate_labels):
@@ -40,6 +41,22 @@ def bbox_converter(row, obj):
     converter.regsistered_actions[obj.feature_schema_id] = bbox_converter
 
 
+    def interpolator(start, end, progress):
+        start_box = start['bounding_box']
+        end_box = end['bounding_box']
+        bbox = copy.deepcopy(start)
+        bbox['bounding_box'] = {
+                'top': start_box['top'] + (end_box['top'] - start_box['top']) * progress,
+                'left': start_box['left'] + (end_box['left'] - start_box['left']) * progress,
+                'width': start_box['width'] + (end_box['width'] - start_box['width']) * progress,
+                'height': start_box['height'] + (end_box['height'] - start_box['height']) * progress,
+            }
+
+        return bbox
+
+    converter.registered_interpolators[obj.feature_schema_id] = interpolator
+
+
 def radio_converter_(obj, converter, tensor_name, context, generate_labels):
     ds = context["ds"]
 
@@ -144,6 +161,19 @@ def point_converter(row, obj):
 
     converter.regsistered_actions[obj.feature_schema_id] = point_converter
 
+    def interpolator(start, end, progress):
+        start_point = start['point']
+        end_point = end['point']
+        point = copy.deepcopy(start)
+        point['point'] = {
+                'x': start_point['x'] + (end_point['x'] - start_point['x']) * progress,
+                'y': start_point['y'] + (end_point['y'] - start_point['y']) * progress,
+            }
+
+        return point
+
+    converter.registered_interpolators[obj.feature_schema_id] = interpolator
+
 
 def line_converter_(obj, converter, tensor_name, context, generate_labels):
     ds = context["ds"]
@@ -167,6 +197,22 @@ def polygon_converter(row, obj):
 
     converter.regsistered_actions[obj.feature_schema_id] = polygon_converter
 
+    def interpolator(start, end, progress):
+        start_line = start['line']
+        end_line = end['line']
+        line = copy.deepcopy(start)
+        line['line'] = [
+            [
+                start_line[i]['x'] + (end_line[i]['x'] - start_line[i]['x']) * progress,
+                start_line[i]['y'] + (end_line[i]['y'] - start_line[i]['y']) * progress,
+            ]
+            for i in range(len(start_line))
+        ]
+
+        return line
+
+    converter.registered_interpolators[obj.feature_schema_id] = interpolator
+
 
 def raster_segmentation_converter_(
     obj, converter, tensor_name, context, generate_labels