Motion detection plug-in model and analysis (#175)

* motion algorithm plug-in model and live analysis

* alter motion algorithm to output analysis frames to capture handler

* remove auto pkg generation (don't break the build!)

* image processing (convolution) using cell-based parallel processing

* restore fixes in commit 8b07ab3

* encoded image support

* add cell counts for padded raw buffer dimensions

* suppress callback if buffer length is zero

* fixes #187

* test that the source data is raw

* copy cloned resolution into original context (TakePicture doesn't set it yet)

src/MMALSharp.Common/ImageContext.cs

@@ -13,54 +13,56 @@ namespace MMALSharp.Common
     /// </summary>
     public class ImageContext
     {
+        // Fields are used rather than properties for hot-path performance reasons.
+
         /// <summary>
         /// The working data.
         /// </summary>
-        public byte[] Data { get; set; }
-        
+        public byte[] Data;
+
         /// <summary>
         /// Flag to indicate whether image frame is raw.
         /// </summary>
-        public bool Raw { get; set; }
+        public bool Raw;
 
         /// <summary>
         /// The resolution of the frame we're processing.
         /// </summary>
-        public Resolution Resolution { get; set; }
+        public Resolution Resolution;
 
         /// <summary>
         /// The encoding format of the frame we're processing.
         /// </summary>
-        public MMALEncoding Encoding { get; set; }
+        public MMALEncoding Encoding;
 
         /// <summary>
         /// The pixel format of the frame we're processing.
         /// </summary>
-        public MMALEncoding PixelFormat { get; set; }
-        
+        public MMALEncoding PixelFormat;
+
         /// <summary>
         /// The image format to store the processed data in.
         /// </summary>
-        public ImageFormat StoreFormat { get; set; }
+        public ImageFormat StoreFormat;
 
         /// <summary>
         /// Indicates if this frame represents the end of the stream.
         /// </summary>
-        public bool Eos { get; set; }
+        public bool Eos;
 
         /// <summary>
         /// Indicates if this frame contains IFrame data.
         /// </summary>
-        public bool IFrame { get; set; }
+        public bool IFrame;
 
         /// <summary>
         /// The timestamp value.
         /// </summary>
-        public long? Pts { get; set; }
-        
+        public long? Pts;
+
         /// <summary>
         /// The pixel format stride.
         /// </summary>
-        public int Stride { get; set; }
+        public int Stride;
     }
 }

src/MMALSharp.Common/MMALSharp.Common.csproj

@@ -17,7 +17,8 @@
 	<VersionPrefix>0.7.0</VersionPrefix>
 	<CodeAnalysisRuleSet>..\..\StyleCop.Analyzers.ruleset</CodeAnalysisRuleSet>	
 	<DocumentationFile>bin\$(Configuration)\$(TargetFramework)\$(AssemblyName).xml</DocumentationFile>
-	<ProjectGuid>{65a1440e-72e1-4943-b469-5cfba8cb5633}</ProjectGuid> <!--Project guid for Sonar-->
+	<ProjectGuid>{65a1440e-72e1-4943-b469-5cfba8cb5633}</ProjectGuid>
+	<!--Project guid for Sonar-->
   </PropertyGroup>
   <PropertyGroup Condition="'$(Platform)'=='AnyCPU'">
     <AllowUnsafeBlocks>True</AllowUnsafeBlocks>

src/MMALSharp.FFmpeg/Handlers/FFmpegCaptureHandler.cs

@@ -74,7 +74,7 @@ public static ExternalProcessCaptureHandler RawVideoToMP4(string directory, stri
             {
                 Filename = "ffmpeg",
                 Arguments = $"-framerate {fps} -i - -b:v {bitrate}k -c copy -movflags +frag_keyframe+separate_moof+omit_tfhd_offset+empty_moov {directory.TrimEnd()}/{filename}.mp4",
-                EchoOutput = true,
+                EchoOutput = echoOutput,
                 DrainOutputDelayMs = 500, // default
                 TerminationSignals = ExternalProcessCaptureHandlerOptions.SignalsFFmpeg
             };

src/MMALSharp.FFmpeg/Handlers/VLCCaptureHandler.cs

@@ -15,25 +15,50 @@ public static class VLCCaptureHandler
         private static readonly string _VLCInternalMimeBoundaryName = "7b3cc56e5f51db803f790dad720ed50a";
 
         /// <summary>
-        /// Listens for a request on the given port and begins streaming MJPEG images when a client connects.
+        /// Listens for a request on the given port and begins streaming MJPEG images when a client connects. Requires h.264 encoded I420 (YUV420p) as input.
         /// </summary>
         /// <param name="listenPort">The port to listen on. Defaults to 8554.</param>
         /// <param name="echoOutput">Whether to echo stdout and stderr to the console or suppress it. Defaults to true.</param>
         /// <param name="maxBitrate">Maximum output bitrate. If source data is available at a higher bitrate, VLC caps to this. Defaults to 2500 (25Mbps).</param>
         /// <param name="maxFps">Maximum output framerate. If source data is available at a higher framerate, VLC caps to this. Defaults to 20.</param>
         /// <returns>An initialized instance of <see cref="ExternalProcessCaptureHandler"/></returns>
-        public static ExternalProcessCaptureHandler StreamMJPEG(int listenPort = 8554, bool echoOutput = true, int maxBitrate = 2500, int maxFps = 20)
+        public static ExternalProcessCaptureHandler StreamH264asMJPEG(int listenPort = 8554, bool echoOutput = true, int maxBitrate = 2500, int maxFps = 20)
         {
             var opts = new ExternalProcessCaptureHandlerOptions
             {
                 Filename = "cvlc",
-                Arguments = $"stream:///dev/stdin --sout \"#transcode{{vcodec=mjpg,vb={maxBitrate},fps={maxFps},acodec=none}}:standard{{access=http{{mime=multipart/x-mixed-replace;boundary=--{_VLCInternalMimeBoundaryName}}},mux=mpjpeg,dst=:{listenPort}/}}\" :demux=h264",
+                Arguments = $"stream:///dev/stdin --sout \"#transcode{{vcodec=mjpg,vb={maxBitrate},fps={maxFps},acodec=none}}:standard{{access=http{{mime=multipart/x-mixed-replace;boundary={_VLCInternalMimeBoundaryName}}},mux=mpjpeg,dst=:{listenPort}/}}\" :demux=h264",
                 EchoOutput = echoOutput,
                 DrainOutputDelayMs = 500, // default
                 TerminationSignals = ExternalProcessCaptureHandlerOptions.SignalsVLC
             };
 
             return new ExternalProcessCaptureHandler(opts);
         }
+
+        /// <summary>
+        /// Listens for a request on the given port and begins streaming MJPEG images when a client connects. Requires raw RGB24 frames as input.
+        /// </summary>
+        /// <param name="width">The width of the raw frames. Defaults to 640.</param>
+        /// <param name="height">The height of the raw frames. Defaults to 480.</param>
+        /// <param name="fps">Expected FPS of the raw frames. Defaults to 24.</param>
+        /// <param name="listenPort">The port to listen on. Defaults to 8554.</param>
+        /// <param name="echoOutput">Whether to echo stdout and stderr to the console or suppress it. Defaults to true.</param>
+        /// <param name="maxBitrate">Maximum output bitrate. If source data is available at a higher bitrate, VLC caps to this. Defaults to 2500 (25Mbps).</param>
+        /// <param name="maxFps">Maximum output framerate. If source data is available at a higher framerate, VLC caps to this. Defaults to 20.</param>
+        /// <returns>An initialized instance of <see cref="ExternalProcessCaptureHandler"/></returns>
+        public static ExternalProcessCaptureHandler StreamRawRGB24asMJPEG(int width = 640, int height = 480, int fps = 24, int listenPort = 8554, bool echoOutput = true, int maxBitrate = 2500, int maxFps = 20)
+        {
+            var opts = new ExternalProcessCaptureHandlerOptions
+            {
+                Filename = "/bin/bash",
+                EchoOutput = true,
+                Arguments = $"-c \"ffmpeg -hide_banner -f rawvideo -c:v rawvideo -pix_fmt rgb24 -s:v {width}x{height} -r {fps} -i - -f h264 -c:v libx264 -preset ultrafast -tune zerolatency -vf format=yuv420p - | cvlc stream:///dev/stdin --sout '#transcode{{vcodec=mjpg,vb={maxBitrate},fps={maxFps},acodec=none}}:standard{{access=http{{mime=multipart/x-mixed-replace;boundary={_VLCInternalMimeBoundaryName}}},mux=mpjpeg,dst=:{listenPort}/}}' :demux=h264\"",
+                DrainOutputDelayMs = 500, // default = 500
+                TerminationSignals = ExternalProcessCaptureHandlerOptions.SignalsFFmpeg
+            };
+
+            return new ExternalProcessCaptureHandler(opts);
+        }
     }
 }

src/MMALSharp.FFmpeg/MMALSharp.FFmpeg.csproj

@@ -17,7 +17,8 @@
 	<VersionPrefix>0.7.0</VersionPrefix>
 	<CodeAnalysisRuleSet>..\..\StyleCop.Analyzers.ruleset</CodeAnalysisRuleSet>	
 	<DocumentationFile>bin\$(Configuration)\$(TargetFramework)\$(AssemblyName).xml</DocumentationFile>
-	<ProjectGuid>{0600c674-e587-4267-89f3-b52ae9591f80}</ProjectGuid> <!--Project guid for Sonar-->
+	<ProjectGuid>{0600c674-e587-4267-89f3-b52ae9591f80}</ProjectGuid>
+	<!--Project guid for Sonar-->
   </PropertyGroup>
   <PropertyGroup Condition="'$(Platform)'=='AnyCPU'">
     <AllowUnsafeBlocks>True</AllowUnsafeBlocks>

src/MMALSharp.Processing/Handlers/ExternalProcessCaptureHandler.cs

@@ -112,15 +112,6 @@ public void PostProcess()
         public string GetDirectory()
             => throw new NotImplementedException();
 
-        /// <summary>
-        /// Not used.
-        /// </summary>
-        /// <param name="allocSize">N/A.</param>
-        /// <returns>A NotImplementedException.</returns>
-        /// <exception cref="NotImplementedException"></exception>
-        public ProcessResult Process(uint allocSize)
-            => throw new NotImplementedException();
-
         /// <summary>
         /// Writes frame data to the StandardInput stream for processing.
         /// </summary>

src/MMALSharp.Processing/Handlers/FrameBufferCaptureHandler.cs

@@ -16,9 +16,8 @@ namespace MMALSharp.Handlers
     /// </summary>
     public class FrameBufferCaptureHandler : MemoryStreamCaptureHandler, IMotionCaptureHandler, IVideoCaptureHandler
     {
-        private MotionConfig _motionConfig;
         private bool _detectingMotion;
-        private FrameDiffAnalyser _motionAnalyser;
+        private FrameDiffDriver _driver;
 
         private bool _waitForFullFrame = true;
         private bool _writeFrameRequested = false;
@@ -45,6 +44,20 @@ public FrameBufferCaptureHandler()
             : base()
         { }
 
+        /// <summary>
+        /// Creates a new <see cref="FrameBufferCaptureHandler"/> configured for motion detection analysis (either using a recorded
+        /// raw video stream where MMALStandalone.Instance is used, or when the camera is used but triggering motion detection events
+        /// is unnecessary). If motion detection events are desired, use the camera's WithMotionDetection method.
+        /// </summary>
+        /// <param name="motionConfig">The motion configuration.</param>
+        /// <param name="onDetect">A callback for when motion is detected.</param>
+        public FrameBufferCaptureHandler(MotionConfig motionConfig, Action onDetect)
+            : base()
+        {
+            _driver = new FrameDiffDriver(motionConfig, onDetect);
+            _detectingMotion = true;
+        }
+
         /// <summary>
         /// Target directory when <see cref="WriteFrame"/> is invoked without a directory argument.
         /// </summary>
@@ -99,7 +112,7 @@ public override void Process(ImageContext context)
 
             if (_detectingMotion)
             {
-                _motionAnalyser.Apply(context);
+                _driver.Apply(context);
             }
 
             // accumulate frame data in the underlying memory stream
@@ -122,22 +135,35 @@ public override void Process(ImageContext context)
         /// <inheritdoc />
         public void ConfigureMotionDetection(MotionConfig config, Action onDetect)
         {
-            _motionConfig = config;
-            _motionAnalyser = new FrameDiffAnalyser(config, onDetect);
+            _driver = new FrameDiffDriver(config, onDetect);
             this.EnableMotionDetection();
         }
 
         /// <inheritdoc />
         public void EnableMotionDetection()
         {
-            _detectingMotion = true;
-            _motionAnalyser?.ResetAnalyser();
+            if(_driver.OnDetectEnabled)
+            {
+                _detectingMotion = true;
+                _driver?.ResetAnalyser();
+            }
+            else
+            {
+                _driver.OnDetectEnabled = true;
+            }
         }
 
         /// <inheritdoc />
-        public void DisableMotionDetection()
+        public void DisableMotionDetection(bool disableCallbackOnly = false)
         {
-            _detectingMotion = false;
+            if(disableCallbackOnly)
+            {
+                _driver.OnDetectEnabled = false;
+            }
+            else
+            {
+                _detectingMotion = false;
+            }
         }
 
         /// <inheritdoc />
@@ -158,5 +184,14 @@ private void WriteStreamToFile()
             this.MostRecentFilename = filename;
             this.MostRecentPathname = pathname;
         }
+
+        // This is used for temporary local-development performance testing. See the
+        // commentedlines before and inside FrameDiffDriver around the Apply method.
+        // public override void Dispose()
+        // {
+        //     long perf = (long)((float)_driver.totalElapsed / _driver.frameCounter);
+        //     Console.WriteLine($"{perf} ms/frame, total {_driver.frameCounter} frames");
+        //     base.Dispose();
+        // }
     }
 }

src/MMALSharp.Processing/Handlers/IMotionCaptureHandler.cs

@@ -33,6 +33,8 @@ public interface IMotionCaptureHandler
         /// <summary>
         /// Disables motion detection. When configured, this will instruct the capture handler not to detect motion.
         /// </summary>
-        void DisableMotionDetection();
+        /// <param name="disableCallbackOnly">When true, motion detection will continue but the OnDetect callback
+        /// will not be invoked. Call <see cref="EnableMotionDetection"/> to re-enable the callback.</param>
+        void DisableMotionDetection(bool disableCallbackOnly = false);
     }
 }

src/MMALSharp.Processing/Handlers/InMemoryCaptureHandler.cs

@@ -22,21 +22,21 @@ public class InMemoryCaptureHandler : OutputCaptureHandler
         /// The working data store.
         /// </summary>
         public List<byte> WorkingData { get; set; }
-               
+
         /// <summary>
         /// Creates a new instance of <see cref="InMemoryCaptureHandler"/>.
         /// </summary>
         public InMemoryCaptureHandler()
         {
             this.WorkingData = new List<byte>();
         }
-        
+
         /// <inheritdoc />
         public override void Dispose()
         {
             MMALLog.Logger.LogInformation($"Successfully processed {Helpers.ConvertBytesToMegabytes(_totalProcessed)}.");
         }
-        
+
         /// <inheritdoc />
         public override void Process(ImageContext context)
         {

src/MMALSharp.Processing/Handlers/OutputCaptureHandler.cs

@@ -64,10 +64,10 @@ public virtual void PostProcess()
         }
 
         /// <summary>
-        /// Allows manipulating of the image frame.
+        /// Allows manipulation of the image frame.
         /// </summary>
         /// <param name="context">A delegate to the manipulation you wish to carry out.</param>
-        /// <param name="storeFormat">The image format to save manipulated files in..</param>
+        /// <param name="storeFormat">The image format to save manipulated files in, or null to return raw data.</param>
         public void Manipulate(Action<IFrameProcessingContext> context, ImageFormat storeFormat)
         {
             this.OnManipulate = context;

src/MMALSharp.Processing/Handlers/StreamCaptureHandler.cs

@@ -23,7 +23,7 @@ public abstract class StreamCaptureHandler<T> : OutputCaptureHandler
         /// A Stream instance that we can process image data to.
         /// </summary>
         public T CurrentStream { get; protected set; }
-        
+
         /// <inheritdoc />
         public override void Process(ImageContext context)
         {
@@ -64,11 +64,11 @@ public override void PostProcess()
                         }
 
                         using (var ms = new MemoryStream(this.ImageContext.Data))
-                        {   
+                        {
                             this.CurrentStream.SetLength(0);
                             this.CurrentStream.Position = 0;
                             ms.CopyTo(this.CurrentStream);
-                        }    
+                        }
                     }
                 }
             }
@@ -77,7 +77,7 @@ public override void PostProcess()
                 MMALLog.Logger.LogWarning($"Something went wrong while processing stream: {e.Message}. {e.InnerException?.Message}. {e.StackTrace}");
             }
         }
-        
+
         /// <inheritdoc />
         public override string TotalProcessed()
         {

src/MMALSharp.Processing/MMALSharp.Processing.csproj

@@ -18,7 +18,8 @@
 	<VersionPrefix>0.7.0</VersionPrefix>
 	<CodeAnalysisRuleSet>..\..\StyleCop.Analyzers.ruleset</CodeAnalysisRuleSet>	
 	<DocumentationFile>bin\$(Configuration)\$(TargetFramework)\$(AssemblyName).xml</DocumentationFile>
-	<ProjectGuid>{dabc9991-56ad-4235-ba86-63def12c261a}</ProjectGuid> <!--Project guid for Sonar-->
+	<ProjectGuid>{dabc9991-56ad-4235-ba86-63def12c261a}</ProjectGuid>
+	<!--Project guid for Sonar-->
   </PropertyGroup>
 
   <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">