Add preliminary EyefulTower dataset support (#2672)

* Add ability to download EyefulTower dataset

* wip before I copy linning's stuff in

* Generate per-resolution cameras.xml

* Generate transforms.json at download

* Fix a couple of quotes

* Use official EyefulTower splits for train and val

* Disable projectaria-tools on windows

* Fix extra imports

* Add a new nerfacto method tund for EyefulTower

* Split eyefultower download into a separate file

* Add some fisheye support for eyeful data

* Reformatted imports to not be dumb

* Apparently this file was missed when formatting originally

* Added 1k resolution scenes

* revert method_configs.py to original values

* Also add 1k exrs

* Address feedback

* Revert changes to pyproject.toml, to be added in a later PR

* Oops, probably shouldn't have gotten rid of awscli ...

* changed glob variable name

* Update tensor_dataclass.py

---------

Co-authored-by: Ethan Weber <[email protected]>
Co-authored-by: Brent Yi <[email protected]>

nerfstudio/data/datamanagers/base_datamanager.py

@@ -471,8 +471,8 @@ def _get_pixel_sampler(self, dataset: TDataset, num_rays_per_batch: int) -> Pixe
             CONSOLE.print("[bold yellow]Warning: Some cameras are equirectangular, but using default pixel sampler.")
 
         fisheye_crop_radius = None
-        if dataset.cameras.metadata is not None and "fisheye_crop_radius" in dataset.cameras.metadata:
-            fisheye_crop_radius = dataset.cameras.metadata["fisheye_crop_radius"]
+        if dataset.cameras.metadata is not None:
+            fisheye_crop_radius = dataset.cameras.metadata.get("fisheye_crop_radius")
 
         return self.config.pixel_sampler.setup(
             is_equirectangular=is_equirectangular,

nerfstudio/data/datamanagers/parallel_datamanager.py

@@ -198,8 +198,15 @@ def _get_pixel_sampler(self, dataset: TDataset, num_rays_per_batch: int) -> Pixe
         is_equirectangular = (dataset.cameras.camera_type == CameraType.EQUIRECTANGULAR.value).all()
         if is_equirectangular.any():
             CONSOLE.print("[bold yellow]Warning: Some cameras are equirectangular, but using default pixel sampler.")
+
+        fisheye_crop_radius = None
+        if dataset.cameras.metadata is not None:
+            fisheye_crop_radius = dataset.cameras.metadata.get("fisheye_crop_radius")
+
         return self.config.pixel_sampler.setup(
-            is_equirectangular=is_equirectangular, num_rays_per_batch=num_rays_per_batch
+            is_equirectangular=is_equirectangular,
+            num_rays_per_batch=num_rays_per_batch,
+            fisheye_crop_radius=fisheye_crop_radius,
         )
 
     def setup_train(self):

nerfstudio/data/dataparsers/nerfstudio_dataparser.py

@@ -290,7 +290,12 @@ def _generate_dataparser_outputs(self, split="train"):
         else:
             distortion_params = torch.stack(distort, dim=0)[idx_tensor]
 
-        metadata = {"fisheye_crop_radius": fisheye_crop_radius} if fisheye_crop_radius is not None else None
+        # Only add fisheye crop radius parameter if the images are actually fisheye, to allow the same config to be used
+        # for both fisheye and non-fisheye datasets.
+        metadata = {}
+        if (camera_type in [CameraType.FISHEYE, CameraType.FISHEYE624]) and (fisheye_crop_radius is not None):
+            metadata["fisheye_crop_radius"] = fisheye_crop_radius
+
         cameras = Cameras(
             fx=fx,
             fy=fy,

nerfstudio/process_data/process_data_utils.py

@@ -58,16 +58,18 @@ class CameraModel(Enum):
 }
 
 
-def list_images(data: Path) -> List[Path]:
+def list_images(data: Path, recursive: bool = False) -> List[Path]:
     """Lists all supported images in a directory
 
     Args:
         data: Path to the directory of images.
+        recursive: Whether to search check nested folders in `data`.
     Returns:
         Paths to images contained in the directory
     """
     allowed_exts = [".jpg", ".jpeg", ".png", ".tif", ".tiff"] + ALLOWED_RAW_EXTS
-    image_paths = sorted([p for p in data.glob("[!.]*") if p.suffix.lower() in allowed_exts])
+    glob_str = "**/[!.]*" if recursive else "[!.]*"
+    image_paths = sorted([p for p in data.glob(glob_str) if p.suffix.lower() in allowed_exts])
     return image_paths
 
 

nerfstudio/scripts/downloads/download_data.py

@@ -29,26 +29,13 @@
 import tyro
 from typing_extensions import Annotated
 
-from nerfstudio.configs.base_config import PrintableConfig
 from nerfstudio.process_data import process_data_utils
+from nerfstudio.scripts.downloads.eyeful_tower import EyefulTowerDownload
+from nerfstudio.scripts.downloads.utils import DatasetDownload
 from nerfstudio.utils import install_checks
 from nerfstudio.utils.scripts import run_command
 
 
-@dataclass
-class DatasetDownload(PrintableConfig):
-    """Download a dataset"""
-
-    capture_name = None
-
-    save_dir: Path = Path("data/")
-    """The directory to save the dataset to"""
-
-    def download(self, save_dir: Path) -> None:
-        """Download the dataset"""
-        raise NotImplementedError
-
-
 @dataclass
 class BlenderDownload(DatasetDownload):
     """Download the blender dataset."""
@@ -555,22 +542,15 @@ def download(self, save_dir: Path) -> None:
     Annotated[SDFstudioDemoDownload, tyro.conf.subcommand(name="sdfstudio")],
     Annotated[NeRFOSRDownload, tyro.conf.subcommand(name="nerfosr")],
     Annotated[Mill19Download, tyro.conf.subcommand(name="mill19")],
+    Annotated[EyefulTowerDownload, tyro.conf.subcommand(name="eyefultower")],
 ]
 
 
 def main(
     dataset: DatasetDownload,
 ):
     """Script to download existing datasets.
-    We currently support the following datasets:
-    - nerfstudio: Growing collection of real-world scenes. Use the `capture_name` argument to specify
-        which capture to download.
-    - blender: Blender synthetic scenes realeased with NeRF.
-    - sitcoms3d: Friends TV show scenes.
-    - record3d: Record3d dataset.
-    - dnerf: D-NeRF dataset.
-    - phototourism: PhotoTourism dataset. Use the `capture_name` argument to specify which capture to download.
-    - mill19: Mill 19 dataset. Use the `capture_name` argument to specify which capture to download.
+    We currently support the datasets listed above in the Commands.
 
     Args:
         dataset: The dataset to download (from).