New config parameter --viewer.codec defaults to VP8, but allows H264

nerfstudio/configs/base_config.py

@@ -23,6 +23,8 @@
 from pathlib import Path
 from typing import Any, List, Optional, Tuple, Type
 
+from typing_extensions import Literal
+
 # model instances
 from nerfstudio.utils import writer
 
@@ -149,3 +151,5 @@ class ViewerConfig(PrintableConfig):
     skip_openrelay: bool = False
     """Avoid using openrelay to communicate with the viewer. Try disabling if you have trouble
     connecting to the viewer"""
+    codec: Literal["H264", "VP8"] = "VP8"
+    """Video codec that viewer will use."""

nerfstudio/viewer/server/utils.py

@@ -63,7 +63,7 @@ def three_js_perspective_camera_focal_length(fov: float, image_height: int):
 
 
 def get_intrinsics_matrix_and_camera_to_world_h(
-    camera_object: Dict[str, Any], image_height: int
+    camera_object: Dict[str, Any], image_height: int, image_width: Optional[int] = None
 ) -> Tuple[torch.Tensor, torch.Tensor]:
     """Returns the camera intrinsics matrix and the camera to world homogeneous matrix.
 
@@ -74,7 +74,8 @@ def get_intrinsics_matrix_and_camera_to_world_h(
     # intrinsics
     fov = camera_object["fov"]
     aspect = camera_object["aspect"]
-    image_width = aspect * image_height
+    if image_width is None:
+        image_width = aspect * image_height
     pp_w = image_width / 2.0
     pp_h = image_height / 2.0
     if (camera_object["camera_type"] == "perspective") | (camera_object["camera_type"] == "fisheye"):

nerfstudio/viewer/server/viewer_utils.py

@@ -689,7 +689,8 @@ async def send_webrtc_answer(self, data):
         video = SingleFrameStreamTrack()
         self.video_tracks.add(video)
         video_sender = pc.addTrack(video)
-        force_codec(pc, video_sender, "video/VP8")
+        print(f"viewer using video/{self.config.codec}")
+        force_codec(pc, video_sender, f"video/{self.config.codec}")
 
         await pc.setRemoteDescription(offer)
         answer = await pc.createAnswer()
@@ -831,6 +832,11 @@ def _calculate_image_res(self, camera_object, is_training: bool) -> Optional[Tup
         if image_width > self.max_resolution:
             image_width = self.max_resolution
             image_height = int(image_width / aspect_ratio)
+        if self.config.codec != "VP8":
+            # force even values to allow hardware encoder usage
+            quantize = 2
+            image_width = int(image_width / quantize) * quantize
+            image_height = int(image_height / quantize) * quantize
         return image_height, image_width
 
     def _process_invalid_output(self, output_type: str) -> str:
@@ -904,7 +910,7 @@ def _render_image_in_viewer(self, camera_object, graph: Model, is_training: bool
             return
 
         intrinsics_matrix, camera_to_world_h = get_intrinsics_matrix_and_camera_to_world_h(
-            camera_object, image_height=image_height
+            camera_object, image_height=image_height, image_width=image_width
         )
 
         camera_to_world = camera_to_world_h[:3, :]