From 1f8ab7fa1ae13bfef5b723f9d48be2a982d6f131 Mon Sep 17 00:00:00 2001 From: Hector Cruz <37127013+hect95@users.noreply.github.com> Date: Tue, 21 May 2024 17:35:44 +0100 Subject: [PATCH] Improve ros bag recording (#4) - `record_rosbag.sh` - `max-cache-size` is significantly increased to avoid loosing data while writing - rosbags are splitted every 10 GiB - use `mcap_cfg.yaml` for configuring storage - Data is not compressed to avoid overheads when writing (`compression: "None"`) - More topics are added for recording, this includes dataspeed DBW report topics - `dev.sh` allows `-p` arg to specity a docker volume where rosbags are going to be saved locally - The script will try to use an in-vehicle only directory and will exit if not found - Set Cyclone DDS as ROS RMW and configure it for high msg throughput - Foxglove bridge now runs with `num_threads:=0` --- Dockerfile | 12 +++ config/mcap_cfg.yaml | 11 +++ config/sensor_topics.txt | 96 ++++++++++++---------- cyclone_dds.xml | 9 ++ dev.sh | 50 +++++++++-- runtime.sh | 7 +- scripts/container_tools/foxglove_bridge.sh | 2 +- scripts/container_tools/record_rosbag.sh | 9 +- 8 files changed, 141 insertions(+), 55 deletions(-) create mode 100644 config/mcap_cfg.yaml create mode 100644 cyclone_dds.xml diff --git a/Dockerfile b/Dockerfile index 8b7770a..e9f17f8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,6 +17,7 @@ RUN apt-get update \ ros-"$ROS_DISTRO"-flir-camera-msgs \ ros-"$ROS_DISTRO"-microstrain-inertial-msgs \ ros-"$ROS_DISTRO"-novatel-gps-msgs \ + ros-"$ROS_DISTRO"-nmea-msgs \ ros-"$ROS_DISTRO"-radar-msgs \ && rm -rf /var/lib/apt/lists/* @@ -24,6 +25,17 @@ RUN apt-get update \ ENV ROS_WS /opt/ros_ws WORKDIR $ROS_WS +# Set cyclone DDS ROS RMW +ENV RMW_IMPLEMENTATION=rmw_cyclonedds_cpp + +COPY ./cyclone_dds.xml $ROS_WS/ + +# Configure Cyclone cfg file +ENV CYCLONEDDS_URI=file://${ROS_WS}/cyclone_dds.xml + +# Enable ROS log colorised output +ENV RCUTILS_COLORIZED_OUTPUT=1 + # Copy tools scripts and config COPY scripts/container_tools $ROS_WS/container_tools COPY config $ROS_WS/config diff --git a/config/mcap_cfg.yaml b/config/mcap_cfg.yaml new file mode 100644 index 0000000..2f6de01 --- /dev/null +++ b/config/mcap_cfg.yaml @@ -0,0 +1,11 @@ +noChunkCRC: false +noAttachmentCRC: false +enableDataCRC: false +noSummaryCRC: false +noChunking: false +noMessageIndex: false +noSummary: false +chunkSize: 786432 +compression: "None" +compressionLevel: "Default" +forceCompression: false diff --git a/config/sensor_topics.txt b/config/sensor_topics.txt index 9fbe777..5b7dad2 100644 --- a/config/sensor_topics.txt +++ b/config/sensor_topics.txt @@ -1,30 +1,4 @@ -/cameras/fsp_l/camera_info -/cameras/fsp_l/control -/cameras/fsp_l/image_raw -/cameras/fsp_l/meta -/cameras/lspf_r/camera_info -/cameras/lspf_r/control -/cameras/lspf_r/image_raw -/cameras/lspf_r/meta -/cameras/lspr_l/camera_info -/cameras/lspr_l/control -/cameras/lspr_l/image_raw -/cameras/lspr_l/meta -/cameras/rsp_l/camera_info -/cameras/rsp_l/control -/cameras/rsp_l/image_raw -/cameras/rsp_l/meta -/cameras/rspf_l/camera_info -/cameras/rspf_l/control -/cameras/rspf_l/image_raw -/cameras/rspf_l/meta -/cameras/rspr_r/camera_info -/cameras/rspr_r/control -/cameras/rspr_r/image_raw -/cameras/rspr_r/meta /diagnostics -/ekf/status -/imu/data /joint_states /output/radar/bumper_front_centre/far/image /output/radar/bumper_front_centre/near/image @@ -42,26 +16,62 @@ /output/radar/roof_right_front/near/image /output/radar/roof_right_rear/far/image /output/radar/roof_right_rear/near/image -/parameter_events /robot_description -/rosout -/sensor/gps/rear/bestpos -/sensor/gps/rear/bestvel -/sensor/gps/rear/corrimudata -/sensor/gps/rear/gps_fix -/sensor/gps/rear/gps_sync -/sensor/gps/rear/imu -/sensor/gps/rear/inscov -/sensor/gps/rear/inspva -/sensor/gps/rear/inspvax -/sensor/gps/rear/insstdev -/sensor/gps/rear/nav_sat_fix -/sensor/gps/rear/psrdop2 +/sensor/camera/fsp_l/camera_info +/sensor/camera/fsp_l/image_raw +/sensor/camera/fsp_l/meta +/sensor/camera/lspf_r/camera_info +/sensor/camera/lspf_r/image_raw +/sensor/camera/lspf_r/meta +/sensor/camera/lspr_l/camera_info +/sensor/camera/lspr_l/image_raw +/sensor/camera/lspr_l/meta +/sensor/camera/rsp_l/camera_info +/sensor/camera/rsp_l/image_raw +/sensor/camera/rsp_l/meta +/sensor/camera/rspf_l/camera_info +/sensor/camera/rspf_l/image_raw +/sensor/camera/rspf_l/meta +/sensor/camera/rspr_r/camera_info +/sensor/camera/rspr_r/image_raw +/sensor/camera/rspr_r/meta +/sensor/gps/bestpos +/sensor/gps/bestvel +/sensor/gps/corrimudata +/sensor/gps/gps_fix +/sensor/gps/gps_sync +/sensor/gps/imu +/sensor/gps/inscov +/sensor/gps/inspva +/sensor/gps/inspvax +/sensor/gps/insstdev +/sensor/gps/nav_sat_fix +/sensor/gps/psrdop2 +/sensor/imu/data +/sensor/imu/ekf/status +/sensor/imu/nmea /sensor/lidar/left/points -/sensor/lidar/left/velodyne_packets /sensor/lidar/right/points -/sensor/lidar/right/velodyne_packets -/sensor/lidar/top/ouster_driver/transition_event /sensor/lidar/top/points /tf /tf_static +/vehicle/brake_info_report +/vehicle/brake_report +/vehicle/dbw_enabled +/vehicle/driver_assist_report +/vehicle/fuel_level_report +/vehicle/gear_report +/vehicle/gps/fix +/vehicle/gps/time +/vehicle/gps/vel +/vehicle/imu/data_raw +/vehicle/misc_1_report +/vehicle/sonar_cloud +/vehicle/steering_report +/vehicle/surround_report +/vehicle/throttle_info_report +/vehicle/throttle_report +/vehicle/tire_pressure_report +/vehicle/twist +/vehicle/wheel_position_report +/vehicle/wheel_speed_report diff --git a/cyclone_dds.xml b/cyclone_dds.xml new file mode 100644 index 0000000..99e60b7 --- /dev/null +++ b/cyclone_dds.xml @@ -0,0 +1,9 @@ + + + + + + + + diff --git a/dev.sh b/dev.sh index 7a9a2c6..dcdd2dd 100755 --- a/dev.sh +++ b/dev.sh @@ -3,21 +3,59 @@ # Build docker dev stage and add local code for live development # ---------------------------------------------------------------- +# Default in-vehicle rosbags directory +ROSBAGS_DIR=/recorded_datasets/edinburgh + +# Function to print usage +usage() { + echo "Usage: dev.sh [-p|--path ]" + exit 1 +} + +# Parse command-line options +while [[ "$#" -gt 0 ]]; do + case $1 in + # Option to specify path + -p|--path) + if [ -n "$2" ]; then + ROSBAGS_DIR="$2" + shift + else + echo "Error: Argument for $1 is missing." + usage + fi + ;; + -h|--help) + usage + ;; + *) + echo "Unknown option: $1" + usage + ;; + esac + shift +done + +# Verify ROSBAGS_DIR exists +if [ ! -d "$ROSBAGS_DIR" ]; then + echo "$ROSBAGS_DIR does not exist! Please provide a valid path to store rosbags" + exit 1 +fi + + # Build docker image up to dev stage DOCKER_BUILDKIT=1 docker build \ - -t av_tools_humble:latest \ + -t av_tools:latest-dev \ -f Dockerfile --target dev . # Get the absolute path of the script SCRIPT_DIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") -# Create a dir to store rosbags -mkdir -p rosbags - # Run docker image with local code volumes for development docker run -it --rm --net host --privileged \ -v /dev/shm:/dev/shm \ - -v $SCRIPT_DIR/rosbags:/opt/ros_ws/rosbags \ + -v $ROSBAGS_DIR:/opt/ros_ws/rosbags \ -v $SCRIPT_DIR/scripts/container_tools:/opt/ros_ws/container_tools \ -v $SCRIPT_DIR/config:/opt/ros_ws/config \ - av_tools_humble:latest + -v /etc/localtime:/etc/localtime:ro \ + av_tools:latest-dev diff --git a/runtime.sh b/runtime.sh index 4934378..387e779 100755 --- a/runtime.sh +++ b/runtime.sh @@ -13,7 +13,7 @@ fi # Build docker image only up to base stage DOCKER_BUILDKIT=1 docker build \ - -t av_tools_humble:latest \ + -t av_tools:latest \ -f Dockerfile --target runtime . # Get the absolute path of the script @@ -25,5 +25,6 @@ mkdir -p $SCRIPT_DIR/rosbags # Run docker image docker run -it --rm --net host --privileged \ -v /dev/shm:/dev/shm \ - -v $SCRIPT_DIR/rosbags:/opt/ros_ws/rosbags \ - av_tools_humble:latest $CMD + -v /recorded_datasets/edinburgh:/opt/ros_ws/rosbags \ + -v /etc/localtime:/etc/localtime:ro \ + av_tools:latest $CMD diff --git a/scripts/container_tools/foxglove_bridge.sh b/scripts/container_tools/foxglove_bridge.sh index 6c2ed0c..460f375 100755 --- a/scripts/container_tools/foxglove_bridge.sh +++ b/scripts/container_tools/foxglove_bridge.sh @@ -1,4 +1,4 @@ #!/bin/bash ros2 launch foxglove_bridge foxglove_bridge_launch.xml \ - address:=172.31.0.1 send_buffer_limit:=1000000000 num_threads:=4 + address:=172.31.0.1 send_buffer_limit:=1000000000 num_threads:=0 diff --git a/scripts/container_tools/record_rosbag.sh b/scripts/container_tools/record_rosbag.sh index 946bb74..f4549f1 100755 --- a/scripts/container_tools/record_rosbag.sh +++ b/scripts/container_tools/record_rosbag.sh @@ -14,7 +14,10 @@ DATE_PREFIX=$(date "+%Y_%m_%d-%H_%M_%S") # Define default behavior if no file is provided if [ "$#" -eq 0 ]; then echo -e "No topics list file provided. ${CYAN}Recording all topics.${NO_COLOR}" - ros2 bag record -s mcap --all -o "$OUTPUT_DIR/${DATE_PREFIX}_sensor_recording" + ros2 bag record -s mcap --all --max-cache-size 5000000000 \ + --storage-config-file "$ROS_WS/config/mcap_cfg.yaml" \ + -b 10740000000 \ + -o "$OUTPUT_DIR/${DATE_PREFIX}_sensor_recording" exit 0 elif [ "$#" -ne 1 ]; then echo "Usage: $0 [path-to-topics-list-file]" @@ -39,6 +42,8 @@ if [ ${#TOPICS[@]} -eq 0 ]; then exit 1 else echo -e "Recording topcis from ${MAGENTA}$TOPICS_LIST_FILE${NO_COLOR}" - ros2 bag record -s mcap --max-cache-size 1048576000 --ignore-leaf-topics \ + ros2 bag record -s mcap --max-cache-size 5000000000 \ + --storage-config-file "$ROS_WS/config/mcap_cfg.yaml" \ + -b 10740000000 \ -o "$OUTPUT_DIR/${DATE_PREFIX}_sensor_recording" "${TOPICS[@]}" fi