From 561df2a0374454289c8b89e58f37ba9bcc1af618 Mon Sep 17 00:00:00 2001 From: Liam Schoneveld Date: Tue, 3 Jan 2023 08:21:05 +0900 Subject: [PATCH 1/5] Render cameras with render times for dynamic nerfs --- nerfstudio/cameras/camera_paths.py | 9 +- .../SidePanel/CameraPanel/CameraPanel.jsx | 189 +++++++++++++++++- .../modules/SidePanel/CameraPanel/curve.js | 5 + scripts/render.py | 4 +- 4 files changed, 204 insertions(+), 3 deletions(-) diff --git a/nerfstudio/cameras/camera_paths.py b/nerfstudio/cameras/camera_paths.py index f5eb675015..85f7ac4728 100644 --- a/nerfstudio/cameras/camera_paths.py +++ b/nerfstudio/cameras/camera_paths.py @@ -109,7 +109,7 @@ def get_spiral_path( ) -def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras: +def get_path_from_json(camera_path: Dict[str, Any], use_render_times: bool = False) -> Cameras: """Takes a camera path dictionary and returns a trajectory as a Camera instance. Args: @@ -152,6 +152,12 @@ def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras: fxs.append(focal_length) fys.append(focal_length) + # Iff ALL cameras in the path have a "time" value, construct Cameras with times + if use_render_times and all("render_time" in camera for camera in camera_path["camera_path"]): + times = torch.tensor([camera["render_time"] for camera in camera_path["camera_path"]]) + else: + times = None + camera_to_worlds = torch.stack(c2ws, dim=0) fx = torch.tensor(fxs) fy = torch.tensor(fys) @@ -162,4 +168,5 @@ def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras: cy=image_height / 2, camera_to_worlds=camera_to_worlds, camera_type=camera_type, + times=times, ) diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx index e74eaf90e9..50b5b07e49 100644 --- a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx +++ b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx @@ -63,6 +63,104 @@ function set_camera_position(camera, matrix) { mat.decompose(camera.position, camera.quaternion, camera.scale); } + +function RenderTimeSelector(props) { + const disabled = props.disabled; + const isGlobal = props.isGlobal; + const camera = props.camera; + const dispatch = props.dispatch; + const globalRenderTime = props.globalRenderTime; + const setGlobalRenderTime = props.setGlobalRenderTime; + const applyAll = props.applyAll; + const changeMain = props.changeMain; + const setAllCameraRenderTime = props.setAllCameraRenderTime; + + const getRenderTimeLabel = () => { + if (!isGlobal) { + return camera.renderTime; + } + camera.renderTime = globalRenderTime + return globalRenderTime; + }; + + const [UIRenderTime, setUIRenderTime] = React.useState( + isGlobal ? globalRenderTime : getRenderTimeLabel(), + ); + + const [valid, setValid] = React.useState(true); + + useEffect( + () => setUIRenderTime(getRenderTimeLabel()), + [camera, globalRenderTime], + ); + + const setRndrTime = (val) => { + if (!isGlobal) { + camera.renderTime = val; + } else { + camera.renderTime = val; + setGlobalRenderTime(val); + } + + if (applyAll) { + setAllCameraRenderTime(val); + } + + if (changeMain) { + dispatch({ + type: 'write', + path: 'renderingState/render_time', + data: camera.renderTime, + }); + } + }; + + const handleValidation = (e) => { + const valueFloat = parseFloat(e.target.value); + let valueStr = String(valueFloat); + if (e.target.value >= 0 && e.target.value <= 1){ + setValid(true); + if (valueFloat === 1.0) { + valueStr = '1.0'; + } + if (valueFloat === 0.0) { + valueStr = '0.0'; + } + setUIRenderTime(valueStr); + setRndrTime(parseFloat(valueStr)); + } else { + setValid(false); + } + }; + + return ( + setUIRenderTime(e.target.value)} + onBlur={(e) => handleValidation(e)} + disabled={disabled} + sx={{ + input: { + '-webkit-text-fill-color': `${ + disabled ? '#24B6FF' : '#EBEBEB' + } !important`, + color: `${disabled ? '#24B6FF' : '#EBEBEB'} !important`, + }, + }} + value={UIRenderTime} + error={!valid} + helperText={!valid ? 'RenderTime should be between 0.0 and 1.0' : ''} + variant="standard" + /> + ); +} + function FovSelector(props) { const fovLabel = props.fovLabel; const setFovLabel = props.setFovLabel; @@ -75,6 +173,7 @@ function FovSelector(props) { const isGlobal = props.isGlobal; const globalFov = props.globalFov; const setGlobalFov = props.setGlobalFov; + const isAnimated = props.isAnimated; const getFovLabel = () => { if (!isGlobal) { @@ -279,6 +378,7 @@ function CameraList(props) { set_camera_position(camera_render, first_camera.matrix); camera_render_helper.set_visibility(true); camera_render.fov = first_camera.fov; + camera_render.renderTime = first_camera.renderTime; } set_slider_value(slider_min); }; @@ -392,6 +492,8 @@ function CameraList(props) { e.stopPropagation(); set_camera_position(camera_main, camera.matrix); camera_main.fov = camera.fov; + camera_main.renderTime = camera.renderTime; + camera_main.renderTime = camera.renderTime; set_slider_value(camera.properties.get('TIME')); }} > @@ -414,7 +516,16 @@ function CameraList(props) { changeMain={false} /> )} - {!isAnimated('FOV') && ( + {isAnimated('RenderTime') && ( + + )} + {!isAnimated('FOV') && !isAnimated('RenderTime') && (

Animated camera properties will show up here!

@@ -452,6 +563,7 @@ export default function CameraPanel(props) { ); const websocket = useContext(WebSocketContext).socket; const DEFAULT_FOV = 50; + const DEFAULT_TIME = '0.0'; // react state const [cameras, setCameras] = React.useState([]); @@ -466,6 +578,7 @@ export default function CameraPanel(props) { const [render_modal_open, setRenderModalOpen] = React.useState(false); const [animate, setAnimate] = React.useState(new Set()); const [globalFov, setGlobalFov] = React.useState(DEFAULT_FOV); + const [globalRenderTime, setGlobalRenderTime] = React.useState(DEFAULT_TIME); // leva store const cameraPropsStore = useCreateStore(); @@ -522,6 +635,14 @@ export default function CameraPanel(props) { }); }; + const setRenderTime = (value) => { + dispatch({ + type: 'write', + path: 'renderingState/render_time', + data: parseFloat(value), + }); + }; + // ui state const [fovLabel, setFovLabel] = React.useState(FOV_LABELS.FOV); @@ -538,6 +659,7 @@ export default function CameraPanel(props) { if (new_camera_list.length >= 1) { set_camera_position(camera_render, new_camera_list[0].matrix); setFieldOfView(new_camera_list[0].fov); + setRenderTime(new_camera_list[0].renderTime) set_slider_value(slider_min); } }; @@ -546,6 +668,7 @@ export default function CameraPanel(props) { const camera_main_copy = camera_main.clone(); camera_main_copy.aspect = 1.0; camera_main_copy.fov = globalFov; + camera_main_copy.renderTime = globalRenderTime; const new_camera_properties = new Map(); camera_main_copy.properties = new_camera_properties; new_camera_properties.set('FOV', globalFov); @@ -825,6 +948,7 @@ export default function CameraPanel(props) { const lookat = curve_object.curve_lookats.getPoint(pt); const up = curve_object.curve_ups.getPoint(pt); const fov = curve_object.curve_fovs.getPoint(pt).z; + const renderTime = curve_object.curve_render_times.getPoint(pt).z; const mat = get_transform_matrix(position, lookat, up); @@ -832,6 +956,7 @@ export default function CameraPanel(props) { camera_to_world: mat.transpose().elements, // convert from col-major to row-major matrix fov, aspect: camera_render.aspect, + render_time: renderTime, }); } @@ -985,6 +1110,23 @@ export default function CameraPanel(props) { } }; + const setAllCameraRenderTime = (val) => { + for (let i = 0; i < cameras.length; i += 1) { + cameras[i].renderTime = val; + } + }; + + const [display_render_time, set_display_render_time] = React.useState(false); + + const receive_temporal_dist = e => { + const msg = msgpack.decode(new Uint8Array(e.data)); + if (msg.path === "/model/has_temporal_distortion") { + set_display_render_time(msg.data === "true"); + websocket.removeEventListener("message", receive_temporal_dist); + } + } + websocket.addEventListener('message', receive_temporal_dist); + return (
@@ -1047,6 +1189,51 @@ export default function CameraPanel(props) { />
+ {display_render_time && ( +
+ + + + +
+ )} {camera_type !== 'equirectangular' && (
diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/curve.js b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/curve.js index 7715a0df96..5cf956f2da 100644 --- a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/curve.js +++ b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/curve.js @@ -30,6 +30,7 @@ export function get_curve_object_from_cameras( const lookats = []; const ups = []; const fovs = []; + const render_times = []; for (let i = 0; i < cameras.length; i += 1) { const camera = cameras[i]; @@ -45,23 +46,27 @@ export function get_curve_object_from_cameras( lookats.push(lookat); // Reuse catmullromcurve3 for 1d values. TODO fix this fovs.push(new THREE.Vector3(0, 0, camera.fov)); + render_times.push(new THREE.Vector3(0, 0, camera.renderTime)); } let curve_positions = null; let curve_lookats = null; let curve_ups = null; let curve_fovs = null; + let curve_render_times = null; curve_positions = get_catmull_rom_curve(positions, is_cycle, smoothness_value); curve_lookats = get_catmull_rom_curve(lookats, is_cycle, smoothness_value); curve_ups = get_catmull_rom_curve(ups, is_cycle, smoothness_value); curve_fovs = get_catmull_rom_curve(fovs, is_cycle, smoothness_value / 10); + curve_render_times = get_catmull_rom_curve(render_times, is_cycle, smoothness_value); const curve_object = { curve_positions, curve_lookats, curve_ups, curve_fovs, + curve_render_times, }; return curve_object; } diff --git a/scripts/render.py b/scripts/render.py index 30b5512287..69ed00afbf 100644 --- a/scripts/render.py +++ b/scripts/render.py @@ -224,6 +224,8 @@ class RenderTrajectory: output_format: Literal["images", "video"] = "video" # Specifies number of rays per chunk during eval. eval_num_rays_per_chunk: Optional[int] = None + # Is this a dynamic NeRF? If so use render_times specified in the camera_path.json + dynamic: bool = False def main(self) -> None: """Main function.""" @@ -259,7 +261,7 @@ def main(self) -> None: camera_type = CameraType.PERSPECTIVE render_width = camera_path["render_width"] render_height = camera_path["render_height"] - camera_path = get_path_from_json(camera_path) + camera_path = get_path_from_json(camera_path, self.dynamic) else: assert_never(self.traj) From 5c0d3c64bb6b104a8999a997a0798c7b0651a93e Mon Sep 17 00:00:00 2001 From: Liam Schoneveld Date: Tue, 3 Jan 2023 20:54:35 +0900 Subject: [PATCH 2/5] Add --dynamic flag to cmd. Clamp render times. --- .../viewer/app/src/modules/RenderModal/RenderModal.jsx | 6 +++++- .../app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx | 6 ++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/nerfstudio/viewer/app/src/modules/RenderModal/RenderModal.jsx b/nerfstudio/viewer/app/src/modules/RenderModal/RenderModal.jsx index 6305d40433..eade7aab27 100644 --- a/nerfstudio/viewer/app/src/modules/RenderModal/RenderModal.jsx +++ b/nerfstudio/viewer/app/src/modules/RenderModal/RenderModal.jsx @@ -8,11 +8,14 @@ import ContentCopyRoundedIcon from '@mui/icons-material/ContentCopyRounded'; interface RenderModalProps { open: object; setOpen: object; + // isDynamic means this is a dynamic nerf and the cmd should include the --dynamic flag + isDynamic: Boolean; } export default function RenderModal(props: RenderModalProps) { const open = props.open; const setOpen = props.setOpen; + const isDynamic = props.isDynamic; // redux store state const config_base_dir = useSelector( @@ -27,7 +30,8 @@ export default function RenderModal(props: RenderModalProps) { // Copy the text inside the text field const config_filename = `${config_base_dir}/config.yml`; const camera_path_filename = `${config_base_dir}/camera_path.json`; - const cmd = `ns-render --load-config ${config_filename} --traj filename --camera-path-filename ${camera_path_filename} --output-path renders/${filename}.mp4`; + const maybe_dynamic_str = isDynamic ? '--dynamic ' : '' + const cmd = `ns-render ${maybe_dynamic_str}--load-config ${config_filename} --traj filename --camera-path-filename ${camera_path_filename} --output-path renders/${filename}.mp4`; const text_intro = `To render a full resolution video, run the following command in a terminal.`; diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx index 50b5b07e49..4cbfd81ca4 100644 --- a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx +++ b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx @@ -173,7 +173,6 @@ function FovSelector(props) { const isGlobal = props.isGlobal; const globalFov = props.globalFov; const setGlobalFov = props.setGlobalFov; - const isAnimated = props.isAnimated; const getFovLabel = () => { if (!isGlobal) { @@ -493,7 +492,6 @@ function CameraList(props) { set_camera_position(camera_main, camera.matrix); camera_main.fov = camera.fov; camera_main.renderTime = camera.renderTime; - camera_main.renderTime = camera.renderTime; set_slider_value(camera.properties.get('TIME')); }} > @@ -956,7 +954,7 @@ export default function CameraPanel(props) { camera_to_world: mat.transpose().elements, // convert from col-major to row-major matrix fov, aspect: camera_render.aspect, - render_time: renderTime, + render_time: Math.max(Math.min(renderTime, 1.0), 0.0), // clamp time values to [0, 1] }); } @@ -1160,7 +1158,7 @@ export default function CameraPanel(props) {

- +

- +