Faster Image Capture

AirLib/include/api/RpcLibAdapatorsBase.hpp

@@ -377,7 +377,7 @@ class RpcLibAdapatorsBase {
             return request;
         }         
     };
-
+    
     struct ImageResponse {
         std::vector<uint8_t> image_data_uint8;
         std::vector<float> image_data_float;
@@ -401,8 +401,8 @@ class RpcLibAdapatorsBase {
         ImageResponse(const msr::airlib::ImageCaptureBase::ImageResponse& s)
         {
             pixels_as_float = s.pixels_as_float;
-            
-            image_data_uint8 = s.image_data_uint8;
+
+            image_data_uint8 = *(s.image_data_uint8);
             image_data_float = s.image_data_float;
 
             //TODO: remove bug workaround for https://github.com/rpclib/rpclib/issues/152
@@ -428,8 +428,8 @@ class RpcLibAdapatorsBase {
 
             d.pixels_as_float = pixels_as_float;
 
-            if (! pixels_as_float)
-                d.image_data_uint8 = image_data_uint8;
+            if (!pixels_as_float)
+                d.image_data_uint8->insert(d.image_data_uint8->begin(), image_data_uint8.front(), image_data_uint8.back());
             else
                 d.image_data_float = image_data_float;
 

AirLib/include/api/VehicleSimApiBase.hpp

@@ -45,8 +45,8 @@ class VehicleSimApiBase : public msr::airlib::UpdatableObject {
 
     virtual void initialize() = 0;
 
-    virtual std::vector<ImageCaptureBase::ImageResponse> getImages(const std::vector<ImageCaptureBase::ImageRequest>& request) const = 0;
-    virtual std::vector<uint8_t> getImage(const std::string& camera_name, ImageCaptureBase::ImageType image_type) const = 0;
+    virtual void getImages(const std::vector<ImageCaptureBase::ImageRequest>& request, std::vector<ImageCaptureBase::ImageResponse> &responses) const = 0;
+    virtual void getImage(const ImageCaptureBase::ImageRequest& request, ImageCaptureBase::ImageResponse &response) const = 0;
 
     virtual Pose getPose() const = 0;
     virtual void setPose(const Pose& pose, bool ignore_collision) = 0;

AirLib/include/common/ImageCaptureBase.hpp

@@ -4,8 +4,10 @@
 #ifndef air_ImageCaptureBase_hpp
 #define air_ImageCaptureBase_hpp
 
+#pragma once
 #include "common/Common.hpp"
 #include "common/common_utils/EnumFlags.hpp"
+#include <functional>
 
 namespace msr { namespace airlib {
 
@@ -44,7 +46,7 @@ class ImageCaptureBase
     };
 
     struct ImageResponse {
-        vector<uint8_t> image_data_uint8;
+        std::unique_ptr<vector<uint8_t>, std::function<void(vector<uint8_t>*)>> image_data_uint8 = nullptr;
         vector<float> image_data_float;
 
         std::string camera_name;
@@ -56,10 +58,49 @@ class ImageCaptureBase
         bool compress = true;
         int width = 0, height = 0;
         ImageType image_type;
+
+        ImageResponse() : image_data_uint8(nullptr), camera_name(""), camera_position(Vector3r::Zero()), camera_orientation(Quaternionr::Identity()), time_stamp(0), message(""), pixels_as_float(false), compress(true), width(0), height(0), image_type(ImageType::Scene) {}
+
+        ImageResponse(const ImageResponse& other)
+        {
+            image_data_uint8 = std::unique_ptr<vector<uint8_t>, std::function<void(vector<uint8_t>*)>>(other.image_data_uint8.get(), std::bind(&ImageResponse::CopyDeleter, this, std::placeholders::_1));
+            camera_name = other.camera_name;
+            camera_position = other.camera_position;
+            camera_orientation = other.camera_orientation;
+            time_stamp = other.time_stamp;
+            message = other.message;
+            pixels_as_float = other.pixels_as_float;
+            compress = other.compress;
+            width = other.width;
+            height = other.height;
+            image_type = other.image_type;
+        }
+
+        ImageResponse& operator=(const ImageResponse& other)
+        {
+            image_data_uint8 = std::unique_ptr<vector<uint8_t>, std::function<void(vector<uint8_t>*)>>(other.image_data_uint8.get(), std::bind(&ImageResponse::CopyDeleter, this, std::placeholders::_1));
+            camera_name = other.camera_name;
+            camera_position = other.camera_position;
+            camera_orientation = other.camera_orientation;
+            time_stamp = other.time_stamp;
+            message = other.message;
+            pixels_as_float = other.pixels_as_float;
+            compress = other.compress;
+            width = other.width;
+            height = other.height;
+            image_type = other.image_type;
+            return *this;
+        }
+
+    private:
+        void CopyDeleter(vector<uint8_t>* buf) {
+            buf = nullptr; //Avoid error for unreferenced formal parameter.
+        } //Copies of an ImageResponse effectively contain weak pointers. The original response handles deleting the buffer. Ultimately, response copying should be removed from everywhere.
     };
 
 public: //methods
     virtual void getImages(const std::vector<ImageRequest>& requests, std::vector<ImageResponse>& responses) const = 0;
+    virtual void getImage(const ImageRequest& request, ImageResponse& response) const = 0;
 };
 
 

AirLib/include/common/common_utils/BufferPool.h

@@ -0,0 +1,37 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+#include <vector>
+#include <map>
+#include <memory>
+#include <unordered_set>
+#include <functional>
+#include <limits>
+
+typedef std::vector<uint8_t> Buffer;
+typedef std::function<void(Buffer*)> Deleter;
+typedef std::unique_ptr<Buffer, Deleter> BufferPtr;
+
+class BufferPool
+{
+public:
+    BufferPtr GetBufferExactSize(size_t size);
+    BufferPtr GetBufferAtLeastSize(size_t size, size_t max_size = std::numeric_limits<size_t>::max());
+
+private:
+    class BufferCollection
+    {
+    public:
+        BufferCollection(size_t size) : Size(size) {}
+        BufferPtr DemandBuffer();
+        BufferPtr GetBufferIfAvailable();
+
+    private:
+        size_t Size;
+        std::unordered_set<Buffer*> AvailableBuffers;
+        BufferPtr MakeBufferPtr(Buffer *underlyingBuffer = nullptr);
+        void ReturnToCollection(Buffer *buffer);
+    };
+    std::map<size_t, BufferCollection> CollectionsBySize;
+};

AirLib/include/common/common_utils/Utils.hpp

@@ -333,7 +333,7 @@ class Utils {
                 }
                 it++;
                 start = it;
-                for (; it != end; it++) {					
+                for (; it != end; it++) {                   
                     if (*it == ch) {
                         break;
                     } 
@@ -585,20 +585,20 @@ class Utils {
         return std::numeric_limits<TReal>::epsilon();
     }
 
-	//implements relative method - do not use for comparing with zero
-	//use this most of the time, tolerance needs to be meaningful in your context
-	template<typename TReal>
-	static bool isApproximatelyEqual(TReal a, TReal b, TReal tolerance = epsilon<TReal>())
-	{
-		TReal diff = std::fabs(a - b);
-		if (diff <= tolerance)
-			return true;
+    //implements relative method - do not use for comparing with zero
+    //use this most of the time, tolerance needs to be meaningful in your context
+    template<typename TReal>
+    static bool isApproximatelyEqual(TReal a, TReal b, TReal tolerance = epsilon<TReal>())
+    {
+        TReal diff = std::fabs(a - b);
+        if (diff <= tolerance)
+            return true;
 
-		if (diff < std::fmax(std::fabs(a), std::fabs(b)) * tolerance)
-			return true;
+        if (diff < std::fmax(std::fabs(a), std::fabs(b)) * tolerance)
+            return true;
 
-		return false;
-	}
+        return false;
+    }
 
     //supply tolerance that is meaningful in your context
     //for example, default tolerance may not work if you are comparing double with float

AirLib/src/api/RpcLibClientBase.cpp

@@ -232,6 +232,7 @@ vector<ImageCaptureBase::ImageResponse> RpcLibClientBase::simGetImages(vector<Im
 
     return RpcLibAdapatorsBase::ImageResponse::to(response_adaptor);
 }
+
 vector<uint8_t> RpcLibClientBase::simGetImage(const std::string& camera_name, ImageCaptureBase::ImageType type, const std::string& vehicle_name)
 {
     vector<uint8_t> result = pimpl_->client.call("simGetImage", camera_name, type, vehicle_name).as<vector<uint8_t>>();

AirLib/src/api/RpcLibServerBase.cpp

@@ -122,18 +122,27 @@ RpcLibServerBase::RpcLibServerBase(ApiProvider* api_provider, const std::string&
     pimpl_->server.bind("armDisarm", [&](bool arm, const std::string& vehicle_name) -> bool { 
         return getVehicleApi(vehicle_name)->armDisarm(arm);
     });
-    pimpl_->server.bind("simGetImages", [&](const std::vector<RpcLibAdapatorsBase::ImageRequest>& request_adapter, const std::string& vehicle_name) -> 
-        vector<RpcLibAdapatorsBase::ImageResponse> {
-            const auto& response = getVehicleSimApi(vehicle_name)->getImages(RpcLibAdapatorsBase::ImageRequest::to(request_adapter));
-            return RpcLibAdapatorsBase::ImageResponse::from(response);
+
+    pimpl_->server.bind("simGetImages", [&](const std::vector<RpcLibAdapatorsBase::ImageRequest>& request_adapter, const std::string& vehicle_name) -> vector<RpcLibAdapatorsBase::ImageResponse> {
+        std::vector<ImageCaptureBase::ImageResponse> responses;
+        for (RpcLibAdapatorsBase::ImageRequest request : request_adapter)
+            responses.push_back(ImageCaptureBase::ImageResponse());
+        getVehicleSimApi(vehicle_name)->getImages(RpcLibAdapatorsBase::ImageRequest::to(request_adapter), responses);
+        return RpcLibAdapatorsBase::ImageResponse::from(responses);
     });
+
     pimpl_->server.bind("simGetImage", [&](const std::string& camera_name, ImageCaptureBase::ImageType type, const std::string& vehicle_name) -> vector<uint8_t> {
-        auto result = getVehicleSimApi(vehicle_name)->getImage(camera_name, type);
-        if (result.size() == 0) {
-            // rpclib has a bug with serializing empty vectors, so we return a 1 byte vector instead.
-            result.push_back(0);
-        }
-        return result;
+
+        ImageCaptureBase::ImageRequest request(camera_name, type);
+        ImageCaptureBase::ImageResponse response;
+
+        getVehicleSimApi(vehicle_name)->getImage(request, response);
+
+        // rpclib has a bug with serializing empty vectors, so we return a 1 byte vector instead.
+        if (response.image_data_uint8->size() == 0)
+            response.image_data_uint8->push_back(0);
+
+        return *response.image_data_uint8;
     });
 
     pimpl_->server.bind("simGetMeshPositionVertexBuffers", [&]() ->vector<RpcLibAdapatorsBase::MeshPositionVertexBuffersResponse> {
@@ -309,9 +318,9 @@ RpcLibServerBase::RpcLibServerBase(ApiProvider* api_provider, const std::string&
         getVehicleApi(vehicle_name)->cancelLastTask();
     });
 
-	pimpl_->server.bind("simSwapTextures", [&](const std::string tag, int tex_id, int component_id, int material_id) -> std::vector<string> {
-		return *getWorldSimApi()->swapTextures(tag, tex_id, component_id, material_id);
-	});
+    pimpl_->server.bind("simSwapTextures", [&](const std::string tag, int tex_id, int component_id, int material_id) -> std::vector<string> {
+        return *getWorldSimApi()->swapTextures(tag, tex_id, component_id, material_id);
+    });
 
     //if we don't suppress then server will bomb out for exceptions raised by any method
     pimpl_->server.suppress_exceptions(true);

AirLib/src/common/common_utils/BufferPool.cpp

@@ -0,0 +1,55 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "common/common_utils/BufferPool.h"
+
+BufferPtr BufferPool::GetBufferExactSize(size_t size)
+{
+    if (CollectionsBySize.count(size) == 0)
+        CollectionsBySize.insert(std::pair<size_t, BufferCollection>(size, BufferCollection(size)));
+
+    return CollectionsBySize.at(size).DemandBuffer();
+}
+
+BufferPtr BufferPool::GetBufferAtLeastSize(size_t size, size_t max_size)
+{
+    BufferPtr buffer = nullptr;
+    auto closestOffering = CollectionsBySize.lower_bound(size);
+    if (closestOffering != CollectionsBySize.end() && closestOffering->first < max_size)
+        buffer = closestOffering->second.GetBufferIfAvailable();
+
+    if (buffer != nullptr)
+        return buffer;
+    return GetBufferExactSize(size);
+}
+
+BufferPtr BufferPool::BufferCollection::DemandBuffer()
+{
+    BufferPtr buf = GetBufferIfAvailable();
+    if (buf != nullptr)
+        return buf;
+    return MakeBufferPtr();
+}
+
+BufferPtr BufferPool::BufferCollection::GetBufferIfAvailable()
+{
+    if (AvailableBuffers.size() == 0)
+        return nullptr;
+
+    Buffer *buffer = *AvailableBuffers.begin();
+    AvailableBuffers.erase(buffer);
+    return MakeBufferPtr(buffer);
+}
+
+BufferPtr BufferPool::BufferCollection::MakeBufferPtr(Buffer *underlyingBuffer)
+{
+    if (underlyingBuffer == nullptr)
+        return std::unique_ptr<Buffer, Deleter>(new Buffer(Size), std::bind(&BufferPool::BufferCollection::ReturnToCollection, this, std::placeholders::_1));
+    else
+        return std::unique_ptr<Buffer, Deleter>(underlyingBuffer, std::bind(&BufferPool::BufferCollection::ReturnToCollection, this, std::placeholders::_1));
+}
+
+void BufferPool::BufferCollection::ReturnToCollection(Buffer *buffer)
+{
+    AvailableBuffers.insert(buffer);
+}

Examples/DataCollection/DataCollectorSGM.h

@@ -204,8 +204,8 @@ class DataCollectorSGM {
                     counter++; 
                     continue;
                 }
-                left_img[idx-counter] = result->response.at(0).image_data_uint8 [idx];
-                right_img[idx-counter] = result->response.at(1).image_data_uint8 [idx];
+                //left_img[idx-counter] = result->response.at(0).image_data_uint8 [idx];
+                //right_img[idx-counter] = result->response.at(1).image_data_uint8 [idx];
             }
 
             //Get SGM disparity and confidence

Examples/DataCollection/StereoImageGenerator.hpp

@@ -144,9 +144,9 @@ class StereoImageGenerator {
     static void processImages(common_utils::ProsumerQueue<ImagesResult>* results)
     {
         while (!results->getIsDone()) {
-            msr::airlib::ClockBase* clock = msr::airlib::ClockFactory::get();
+            //msr::airlib::ClockBase* clock = msr::airlib::ClockFactory::get();
 
-            ImagesResult result;
+            /*ImagesResult result;
             if (!results->tryPop(result)) {
                 clock->sleep_for(1);
                 continue;
@@ -184,7 +184,7 @@ class StereoImageGenerator {
                 << " ori:" << VectorMath::toString(result.orientation)
                 << " render time " << result.render_time * 1E3f << "ms" 
                 << " process time " << clock->elapsedSince(process_time) * 1E3f << " ms"
-                << std::endl;
+                << std::endl;*/
 
         }
     }

Examples/DepthNav/DepthNav.hpp

@@ -157,8 +157,8 @@ class DepthNav {
             if (response.at(1).height != params_.depth_height || response.at(1).width != params_.depth_height)
                 throw DepthNavException("Image Dimension mismatch. Please check right camera in the AirSim config file.");
             */
-            const std::vector<uint8_t>& left_image = response.at(0).image_data_uint8;
-            const std::vector<uint8_t>& right_image = response.at(1).image_data_uint8;
+            const std::vector<uint8_t> left_image(response.at(0).image_data_uint8->begin(), response.at(0).image_data_uint8->end());
+            const std::vector<uint8_t> right_image(response.at(1).image_data_uint8->begin(), response.at(1).image_data_uint8->end());
 
             //baseline * focal_length = depth * disparity
             float f = params_.depth_width / (2 * tan(params_.fov/2));

HelloCar/main.cpp

@@ -44,7 +44,7 @@ int main()
             std::getline(std::cin, path);
 
             for (const ImageResponse& image_info : response) {
-                std::cout << "Image uint8 size: " << image_info.image_data_uint8.size() << std::endl;
+                std::cout << "Image uint8 size: " << image_info.image_data_uint8->size() << std::endl;
                 std::cout << "Image float size: " << image_info.image_data_float.size() << std::endl;
 
                 if (path != "") {
@@ -55,7 +55,7 @@ int main()
                     }
                     else {
                         std::ofstream file(file_path + ".png", std::ios::binary);
-                        file.write(reinterpret_cast<const char*>(image_info.image_data_uint8.data()), image_info.image_data_uint8.size());
+                        file.write(reinterpret_cast<const char*>(image_info.image_data_uint8->data()), image_info.image_data_uint8->size());
                         file.close();
                     }
                 }

HelloDrone/main.cpp

@@ -38,7 +38,7 @@ int main()
             std::getline(std::cin, path);
 
             for (const ImageResponse& image_info : response) {
-                std::cout << "Image uint8 size: " << image_info.image_data_uint8.size() << std::endl;
+                std::cout << "Image uint8 size: " << image_info.image_data_uint8->size() << std::endl;
                 std::cout << "Image float size: " << image_info.image_data_float.size() << std::endl;
 
                 if (path != "") {
@@ -49,7 +49,7 @@ int main()
                     }
                     else {
                         std::ofstream file(file_path + ".png", std::ios::binary);
-                        file.write(reinterpret_cast<const char*>(image_info.image_data_uint8.data()), image_info.image_data_uint8.size());
+                        file.write(reinterpret_cast<const char*>(image_info.image_data_uint8->data()), image_info.image_data_uint8->size());
                         file.close();
                     }
                 }