Add BEVFusion-R (#443)

.gitignore

@@ -127,3 +127,8 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# Models and data
+models/*
+data/*
+runs/*

configs/nuscenes/default.yaml

@@ -183,6 +183,9 @@ train_pipeline:
       - lidar2image
       - img_aug_matrix
       - lidar_aug_matrix
+  -
+    type: GTDepth
+    keyframe_only: true 
 
 test_pipeline:
   -
@@ -256,6 +259,9 @@ test_pipeline:
       - lidar2image
       - img_aug_matrix
       - lidar_aug_matrix
+  -
+    type: GTDepth
+    keyframe_only: true 
 
 data:
   samples_per_gpu: 4

configs/nuscenes/det/centerhead/default.yaml

@@ -1,3 +1,5 @@
+gt_paste_stop_epoch: 15
+
 model:
   heads:
     object:

configs/nuscenes/det/centerhead/lssfpn/camera+radar/default.yaml

@@ -0,0 +1,72 @@
+
+data:
+  train:
+    dataset:
+      ann_file: nuscenes_radar/nuscenes_radar_infos_train_radar.pkl
+  val:
+    ann_file: nuscenes_radar/nuscenes_radar_infos_val_radar.pkl
+  test:
+    ann_file: nuscenes_radar/nuscenes_radar_infos_val_radar.pkl
+
+augment2d:
+  resize: [[0.38, 0.55], [0.48, 0.48]]
+
+augment3d:
+  scale: [0.9, 1.1]
+  rotate: [0, 0]
+  translate: 0.5
+
+model:
+  encoders:
+    lidar: null
+    camera:
+      vtransform:
+        type: LSSTransform
+        image_size: ${image_size}
+        xbound: [-51.2, 51.2, 0.8]
+        ybound: [-51.2, 51.2, 0.8]
+        zbound: [-10.0, 10.0, 20.0]
+        dbound: [1.0, 60.0, 1.0]
+    radar:
+      voxelize_reduce: false
+      voxelize:
+        max_num_points: 20
+        point_cloud_range: ${point_cloud_range}
+        voxel_size: ${radar_voxel_size}
+        max_voxels: [30000, 60000]
+      backbone:
+        type: RadarEncoder
+        pts_voxel_encoder:
+          type: RadarFeatureNet
+          in_channels: 45
+          feat_channels: [128, 128, 128, 64]
+          with_distance: false
+          point_cloud_range: ${point_cloud_range}
+          voxel_size: ${radar_voxel_size}
+          norm_cfg:
+            type: BN1d
+            eps: 1.0e-3
+            momentum: 0.01
+        pts_middle_encoder:
+          type: PointPillarsScatter
+          in_channels: 64
+          output_shape: [128, 128]
+        pts_bev_encoder: null
+  heads:
+    object:
+      test_cfg:
+        nms_type:
+          - circle
+          - rotate
+          - rotate
+          - circle
+          - rotate
+          - rotate
+        nms_scale:
+          - [1.0]
+          - [1.0, 1.0]
+          - [1.0, 1.0]
+          - [1.0]
+          - [1.0, 1.0]
+          - [2.5, 4.0]
+

configs/nuscenes/det/centerhead/lssfpn/camera+radar/resnet50/default.yaml

@@ -0,0 +1,83 @@
+
+
+image_size: [256, 704]
+
+model:
+  encoders:
+    camera:
+      backbone:
+        type: ResNet
+        depth: 50
+        num_stages: 4
+        out_indices: [0, 1, 2, 3] 
+        norm_cfg:
+          type: BN2d
+          requires_grad: true
+        norm_eval: false
+        init_cfg:
+          type: Pretrained
+          checkpoint: torchvision://resnet50
+      neck:
+        type: SECONDFPN
+        in_channels: [256, 512, 1024, 2048]
+        out_channels: [128, 128, 128, 128]
+        upsample_strides: [0.25, 0.5, 1, 2]    
+      vtransform:
+        type: LSSTransform
+        in_channels: 512
+        out_channels: 64
+        image_size: ${image_size}
+        feature_size: ${[image_size[0] // 16, image_size[1] // 16]}
+        xbound: [-51.2, 51.2, 0.8]
+        ybound: [-51.2, 51.2, 0.8]
+        zbound: [-10.0, 10.0, 20.0]
+        dbound: [1.0, 60.0, 1.0]
+        downsample: 1
+  decoder:
+    backbone:
+      type: GeneralizedResNet
+      in_channels: 64
+      blocks:
+        - [2, 128, 2]
+        - [2, 256, 2]
+        - [2, 512, 1]
+    neck:
+      type: LSSFPN
+      in_indices: [-1, 0]
+      in_channels: [512, 128]
+      out_channels: 256
+      scale_factor: 2
+  heads:
+    object:
+      train_cfg:
+        code_weights: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]
+  fuser:
+    type: ConvFuser
+    in_channels: [64, 64]
+    out_channels: 64
+
+
+optimizer:
+  paramwise_cfg:
+    custom_keys:
+      absolute_pos_embed: 
+        decay_mult: 0
+      relative_position_bias_table:
+        decay_mult: 0
+      # encoders.camera.backbone:
+      #   lr_mult: 0.1
+
+
+# lr_config:
+#   policy: cyclic
+#   target_ratio: 5.0
+#   cyclic_times: 1
+#   step_ratio_up: 0.4
+
+# momentum_config:
+#   policy: cyclic
+#   cyclic_times: 1
+#   step_ratio_up: 0.4
+
+data:
+  samples_per_gpu: 4

configs/nuscenes/det/centerhead/lssfpn/camera+radar/resnet50/dlss.yaml

@@ -0,0 +1,13 @@
+
+model:
+  encoders:
+    camera:
+      vtransform:
+        type: AwareDBEVDepth
+        bevdepth_downsample: 16
+        bevdepth_refine: false
+        depth_loss_factor: 3.0
+        use_points: radar
+        depth_input: one-hot
+        height_expand: true
+        add_depth_features: true

configs/nuscenes/det/centerhead/lssfpn/camera/256x704/resnet/bevdepth.yaml

@@ -0,0 +1,16 @@
+model:
+  encoders:
+    camera:
+      vtransform:
+        type: AwareBEVDepth
+        bevdepth_downsample: 16
+        bevdepth_refine: false
+        depth_loss_factor: 3.0
+        in_channels: 512
+        out_channels: 64
+        feature_size: ${[image_size[0] // 16, image_size[1] // 16]}
+        xbound: [-51.2, 51.2, 0.8]
+        ybound: [-51.2, 51.2, 0.8]
+        zbound: [-10.0, 10.0, 20.0]
+        dbound: [1.0, 60.0, 1.0]
+        downsample: 1

configs/nuscenes/det/centerhead/lssfpn/camera/256x704/resnet/default.yaml

@@ -0,0 +1,45 @@
+model:
+  encoders:
+    camera:
+      backbone:
+        type: ResNet
+        depth: 50
+        num_stages: 4
+        out_indices: [0, 1, 2, 3]
+        norm_cfg:
+          type: BN2d
+          requires_grad: true
+        norm_eval: false
+        init_cfg:
+          type: Pretrained
+          checkpoint: torchvision://resnet50
+      neck:
+        type: SECONDFPN
+        in_channels: [256, 512, 1024, 2048]
+        out_channels: [128, 128, 128, 128]
+        upsample_strides: [0.25, 0.5, 1, 2]    
+      vtransform:
+        type: LSSTransform
+        in_channels: 512
+        out_channels: 64
+        image_size: ${image_size}
+        feature_size: ${[image_size[0] // 16, image_size[1] // 16]}
+        xbound: [-51.2, 51.2, 0.8]
+        ybound: [-51.2, 51.2, 0.8]
+        zbound: [-10.0, 10.0, 20.0]
+        dbound: [1.0, 60.0, 1.0]
+        downsample: 1
+  decoder:
+    backbone:
+      type: GeneralizedResNet
+      in_channels: 64
+      blocks:
+        - [2, 128, 2]
+        - [2, 256, 2]
+        - [2, 512, 1]
+    neck:
+      type: LSSFPN
+      in_indices: [-1, 0]
+      in_channels: [512, 128]
+      out_channels: 256
+      scale_factor: 2

configs/nuscenes/det/centerhead/lssfpn/camera/256x704/swint/default.yaml

@@ -82,4 +82,4 @@ momentum_config:
   step_ratio_up: 0.4
 
 data:
-  samples_per_gpu: 6
+  samples_per_gpu: 4

configs/nuscenes/det/centerhead/lssfpn/camera/default.yaml

@@ -1,3 +1,4 @@
+
 augment3d:
   scale: [0.95, 1.05]
   rotate: [-0.3925, 0.3925]
@@ -31,5 +32,3 @@ model:
           - [1.0]
           - [1.0, 1.0]
           - [2.5, 4.0]
-
-lr_config: null

configs/nuscenes/det/centerhead/lssfpn/default.yaml

@@ -1,3 +1,13 @@
+
+# data:
+#   train:
+#     dataset:
+#       ann_file: nuscenes_radar/nuscenes_radar_infos_train_radar.pkl
+#   val:
+#     ann_file: nuscenes_radar/nuscenes_radar_infos_val_radar.pkl
+#   test:
+#     ann_file: nuscenes_radar/nuscenes_radar_infos_val_radar.pkl
+
 model:
   decoder:
     backbone:
@@ -21,6 +31,12 @@ optimizer:
   type: AdamW
   lr: 2.0e-4
   weight_decay: 0.01
+  paramwise_cfg:
+    custom_keys:
+      absolute_pos_embed: 
+        decay_mult: 0
+      relative_position_bias_table:
+        decay_mult: 0
 
 optimizer_config:
   grad_clip:
@@ -32,4 +48,4 @@ lr_config:
   warmup: linear
   warmup_iters: 500
   warmup_ratio: 0.33333333
-  min_lr_ratio: 1.0e-3
+  min_lr_ratio: 1.0e-3