Motion detection plug-in model and analysis

src/MMALSharp.Common/ImageContext.cs

@@ -13,54 +13,56 @@ namespace MMALSharp.Common
     /// </summary>
     public class ImageContext
     {
+        // Fields are used rather than properties for hot-path performance reasons.
+
         /// <summary>
         /// The working data.
         /// </summary>
-        public byte[] Data { get; set; }
-        
+        public byte[] Data;
+
         /// <summary>
         /// Flag to indicate whether image frame is raw.
         /// </summary>
-        public bool Raw { get; set; }
+        public bool Raw;
 
         /// <summary>
         /// The resolution of the frame we're processing.
         /// </summary>
-        public Resolution Resolution { get; set; }
+        public Resolution Resolution;
 
         /// <summary>
         /// The encoding format of the frame we're processing.
         /// </summary>
-        public MMALEncoding Encoding { get; set; }
+        public MMALEncoding Encoding;
 
         /// <summary>
         /// The pixel format of the frame we're processing.
         /// </summary>
-        public MMALEncoding PixelFormat { get; set; }
-        
+        public MMALEncoding PixelFormat;
+
         /// <summary>
         /// The image format to store the processed data in.
         /// </summary>
-        public ImageFormat StoreFormat { get; set; }
+        public ImageFormat StoreFormat;
 
         /// <summary>
         /// Indicates if this frame represents the end of the stream.
         /// </summary>
-        public bool Eos { get; set; }
+        public bool Eos;
 
         /// <summary>
         /// Indicates if this frame contains IFrame data.
         /// </summary>
-        public bool IFrame { get; set; }
+        public bool IFrame;
 
         /// <summary>
         /// The timestamp value.
         /// </summary>
-        public long? Pts { get; set; }
-        
+        public long? Pts;
+
         /// <summary>
         /// The pixel format stride.
         /// </summary>
-        public int Stride { get; set; }
+        public int Stride;
     }
 }

src/MMALSharp.Common/MMALSharp.Common.csproj

@@ -17,7 +17,8 @@
 	<VersionPrefix>0.7.0</VersionPrefix>
 	<CodeAnalysisRuleSet>..\..\StyleCop.Analyzers.ruleset</CodeAnalysisRuleSet>	
 	<DocumentationFile>bin\$(Configuration)\$(TargetFramework)\$(AssemblyName).xml</DocumentationFile>
-	<ProjectGuid>{65a1440e-72e1-4943-b469-5cfba8cb5633}</ProjectGuid> <!--Project guid for Sonar-->
+	<ProjectGuid>{65a1440e-72e1-4943-b469-5cfba8cb5633}</ProjectGuid>
+	<!--Project guid for Sonar-->
   </PropertyGroup>
   <PropertyGroup Condition="'$(Platform)'=='AnyCPU'">
     <AllowUnsafeBlocks>True</AllowUnsafeBlocks>

src/MMALSharp.Common/Utility/FormatRawImageExtension.cs

@@ -0,0 +1,84 @@
+// <copyright file="FormatRawImageExtension.cs" company="Techyian">
+// Copyright (c) Ian Auty and contributors. All rights reserved.
+// Licensed under the MIT License. Please see LICENSE.txt for License info.
+// </copyright>
+
+using System;
+using System.Drawing;
+using System.Drawing.Imaging;
+using System.IO;
+using System.Runtime.InteropServices;
+
+namespace MMALSharp.Common.Utility
+{
+    /// <summary>
+    /// An extension to convert raw <see cref="ImageContext"/> data to a bitmap <see cref="ImageFormat"/>.
+    /// </summary>
+    public static class FormatRawImageExtension
+    {
+        /// <summary>
+        /// Converts the raw image data in an <see cref="ImageContext"/> object to a specified bitmap
+        /// <see cref="ImageFormat"/> (such as <see cref="ImageFormat.Jpeg"/> or <see cref="ImageFormat.Png"/>).
+        /// </summary>
+        /// <param name="context">The image to convert. Must be raw data.</param>
+        public static void FormatRawImage(this ImageContext context)
+        {
+            if (!context.Raw)
+            {
+                throw new Exception("ImageContext does not contain raw data");
+            }
+
+            if (context.StoreFormat == null)
+            {
+                throw new Exception("ImageContext.StoreFormat does not define a target ImageFormat");
+            }
+
+            var pixfmt = MMALEncodingToPixelFormat(context.PixelFormat);
+
+            using (var bitmap = new Bitmap(context.Resolution.Width, context.Resolution.Height, pixfmt))
+            {
+                BitmapData bmpData = null;
+                try
+                {
+                    bmpData = bitmap.LockBits(new Rectangle(0, 0, bitmap.Width, bitmap.Height), ImageLockMode.WriteOnly, bitmap.PixelFormat);
+                    var ptr = bmpData.Scan0;
+                    int size = bmpData.Stride * bitmap.Height;
+                    var data = context.Data;
+                    Marshal.Copy(data, 0, ptr, size);
+                }
+                finally
+                {
+                    bitmap.UnlockBits(bmpData);
+                }
+
+                using(var ms = new MemoryStream())
+                {
+                    bitmap.Save(ms, context.StoreFormat);
+                    context.Data = new byte[ms.Length];
+                    Array.Copy(ms.ToArray(), 0, context.Data, 0, ms.Length);
+                }
+            }
+        }
+
+        private static PixelFormat MMALEncodingToPixelFormat(MMALEncoding encoding)
+        {
+            if(encoding == MMALEncoding.RGB24)
+            {
+                return PixelFormat.Format24bppRgb;
+            }
+
+            if (encoding == MMALEncoding.RGB32)
+            {
+                return PixelFormat.Format32bppRgb;
+            }
+
+            if (encoding == MMALEncoding.RGBA)
+            {
+                return PixelFormat.Format32bppArgb;
+            }
+
+            throw new Exception("Unsupported encoding / pixel format");
+        }
+
+    }
+}

src/MMALSharp.FFmpeg/Handlers/FFmpegCaptureHandler.cs

@@ -74,7 +74,7 @@ public static ExternalProcessCaptureHandler RawVideoToMP4(string directory, stri
             {
                 Filename = "ffmpeg",
                 Arguments = $"-framerate {fps} -i - -b:v {bitrate}k -c copy -movflags +frag_keyframe+separate_moof+omit_tfhd_offset+empty_moov {directory.TrimEnd()}/{filename}.mp4",
-                EchoOutput = true,
+                EchoOutput = echoOutput,
                 DrainOutputDelayMs = 500, // default
                 TerminationSignals = ExternalProcessCaptureHandlerOptions.SignalsFFmpeg
             };

src/MMALSharp.FFmpeg/Handlers/VLCCaptureHandler.cs

@@ -15,25 +15,50 @@ public static class VLCCaptureHandler
         private static readonly string _VLCInternalMimeBoundaryName = "7b3cc56e5f51db803f790dad720ed50a";
 
         /// <summary>
-        /// Listens for a request on the given port and begins streaming MJPEG images when a client connects.
+        /// Listens for a request on the given port and begins streaming MJPEG images when a client connects. Requires h.264 encoded I420 (YUV420p) as input.
         /// </summary>
         /// <param name="listenPort">The port to listen on. Defaults to 8554.</param>
         /// <param name="echoOutput">Whether to echo stdout and stderr to the console or suppress it. Defaults to true.</param>
         /// <param name="maxBitrate">Maximum output bitrate. If source data is available at a higher bitrate, VLC caps to this. Defaults to 2500 (25Mbps).</param>
         /// <param name="maxFps">Maximum output framerate. If source data is available at a higher framerate, VLC caps to this. Defaults to 20.</param>
         /// <returns>An initialized instance of <see cref="ExternalProcessCaptureHandler"/></returns>
-        public static ExternalProcessCaptureHandler StreamMJPEG(int listenPort = 8554, bool echoOutput = true, int maxBitrate = 2500, int maxFps = 20)
+        public static ExternalProcessCaptureHandler StreamH264asMJPEG(int listenPort = 8554, bool echoOutput = true, int maxBitrate = 2500, int maxFps = 20)
         {
             var opts = new ExternalProcessCaptureHandlerOptions
             {
                 Filename = "cvlc",
-                Arguments = $"stream:///dev/stdin --sout \"#transcode{{vcodec=mjpg,vb={maxBitrate},fps={maxFps},acodec=none}}:standard{{access=http{{mime=multipart/x-mixed-replace;boundary=--{_VLCInternalMimeBoundaryName}}},mux=mpjpeg,dst=:{listenPort}/}}\" :demux=h264",
+                Arguments = $"stream:///dev/stdin --sout \"#transcode{{vcodec=mjpg,vb={maxBitrate},fps={maxFps},acodec=none}}:standard{{access=http{{mime=multipart/x-mixed-replace;boundary={_VLCInternalMimeBoundaryName}}},mux=mpjpeg,dst=:{listenPort}/}}\" :demux=h264",
                 EchoOutput = echoOutput,
                 DrainOutputDelayMs = 500, // default
                 TerminationSignals = ExternalProcessCaptureHandlerOptions.SignalsVLC
             };
 
             return new ExternalProcessCaptureHandler(opts);
         }
+
+        /// <summary>
+        /// Listens for a request on the given port and begins streaming MJPEG images when a client connects. Requires raw RGB24 frames as input.
+        /// </summary>
+        /// <param name="width">The width of the raw frames. Defaults to 640.</param>
+        /// <param name="height">The height of the raw frames. Defaults to 480.</param>
+        /// <param name="fps">Expected FPS of the raw frames. Defaults to 24.</param>
+        /// <param name="listenPort">The port to listen on. Defaults to 8554.</param>
+        /// <param name="echoOutput">Whether to echo stdout and stderr to the console or suppress it. Defaults to true.</param>
+        /// <param name="maxBitrate">Maximum output bitrate. If source data is available at a higher bitrate, VLC caps to this. Defaults to 2500 (25Mbps).</param>
+        /// <param name="maxFps">Maximum output framerate. If source data is available at a higher framerate, VLC caps to this. Defaults to 20.</param>
+        /// <returns>An initialized instance of <see cref="ExternalProcessCaptureHandler"/></returns>
+        public static ExternalProcessCaptureHandler StreamRawRGB24asMJPEG(int width = 640, int height = 480, int fps = 24, int listenPort = 8554, bool echoOutput = true, int maxBitrate = 2500, int maxFps = 20)
+        {
+            var opts = new ExternalProcessCaptureHandlerOptions
+            {
+                Filename = "/bin/bash",
+                EchoOutput = true,
+                Arguments = $"-c \"ffmpeg -hide_banner -f rawvideo -c:v rawvideo -pix_fmt rgb24 -s:v {width}x{height} -r {fps} -i - -f h264 -c:v libx264 -preset ultrafast -tune zerolatency -vf format=yuv420p - | cvlc stream:///dev/stdin --sout '#transcode{{vcodec=mjpg,vb={maxBitrate},fps={maxFps},acodec=none}}:standard{{access=http{{mime=multipart/x-mixed-replace;boundary={_VLCInternalMimeBoundaryName}}},mux=mpjpeg,dst=:{listenPort}/}}' :demux=h264\"",
+                DrainOutputDelayMs = 500, // default = 500
+                TerminationSignals = ExternalProcessCaptureHandlerOptions.SignalsFFmpeg
+            };
+
+            return new ExternalProcessCaptureHandler(opts);
+        }
     }
 }

src/MMALSharp.FFmpeg/MMALSharp.FFmpeg.csproj

@@ -17,7 +17,8 @@
 	<VersionPrefix>0.7.0</VersionPrefix>
 	<CodeAnalysisRuleSet>..\..\StyleCop.Analyzers.ruleset</CodeAnalysisRuleSet>	
 	<DocumentationFile>bin\$(Configuration)\$(TargetFramework)\$(AssemblyName).xml</DocumentationFile>
-	<ProjectGuid>{0600c674-e587-4267-89f3-b52ae9591f80}</ProjectGuid> <!--Project guid for Sonar-->
+	<ProjectGuid>{0600c674-e587-4267-89f3-b52ae9591f80}</ProjectGuid>
+	<!--Project guid for Sonar-->
   </PropertyGroup>
   <PropertyGroup Condition="'$(Platform)'=='AnyCPU'">
     <AllowUnsafeBlocks>True</AllowUnsafeBlocks>

src/MMALSharp.Processing/Handlers/ExternalProcessCaptureHandler.cs

@@ -112,15 +112,6 @@ public void PostProcess()
         public string GetDirectory()
             => throw new NotImplementedException();
 
-        /// <summary>
-        /// Not used.
-        /// </summary>
-        /// <param name="allocSize">N/A.</param>
-        /// <returns>A NotImplementedException.</returns>
-        /// <exception cref="NotImplementedException"></exception>
-        public ProcessResult Process(uint allocSize)
-            => throw new NotImplementedException();
-
         /// <summary>
         /// Writes frame data to the StandardInput stream for processing.
         /// </summary>

src/MMALSharp.Processing/Handlers/FrameBufferCaptureHandler.cs

@@ -16,9 +16,8 @@ namespace MMALSharp.Handlers
     /// </summary>
     public class FrameBufferCaptureHandler : MemoryStreamCaptureHandler, IMotionCaptureHandler, IVideoCaptureHandler
     {
-        private MotionConfig _motionConfig;
         private bool _detectingMotion;
-        private FrameDiffAnalyser _motionAnalyser;
+        private FrameDiffDriver _driver;
 
         private bool _waitForFullFrame = true;
         private bool _writeFrameRequested = false;
@@ -45,6 +44,20 @@ public FrameBufferCaptureHandler()
             : base()
         { }
 
+        /// <summary>
+        /// Creates a new <see cref="FrameBufferCaptureHandler"/> configured for motion detection analysis (either using a recorded
+        /// raw video stream where MMALStandalone.Instance is used, or when the camera is used but triggering motion detection events
+        /// is unnecessary). If motion detection events are desired, use the camera's WithMotionDetection method.
+        /// </summary>
+        /// <param name="motionConfig">The motion configuration.</param>
+        /// <param name="onDetect">A callback for when motion is detected.</param>
+        public FrameBufferCaptureHandler(MotionConfig motionConfig, Action onDetect)
+            : base()
+        {
+            _driver = new FrameDiffDriver(motionConfig, onDetect);
+            _detectingMotion = true;
+        }
+
         /// <summary>
         /// Target directory when <see cref="WriteFrame"/> is invoked without a directory argument.
         /// </summary>
@@ -99,7 +112,7 @@ public override void Process(ImageContext context)
 
             if (_detectingMotion)
             {
-                _motionAnalyser.Apply(context);
+                _driver.Apply(context);
             }
 
             // accumulate frame data in the underlying memory stream
@@ -122,22 +135,35 @@ public override void Process(ImageContext context)
         /// <inheritdoc />
         public void ConfigureMotionDetection(MotionConfig config, Action onDetect)
         {
-            _motionConfig = config;
-            _motionAnalyser = new FrameDiffAnalyser(config, onDetect);
+            _driver = new FrameDiffDriver(config, onDetect);
             this.EnableMotionDetection();
         }
 
         /// <inheritdoc />
         public void EnableMotionDetection()
         {
-            _detectingMotion = true;
-            _motionAnalyser?.ResetAnalyser();
+            if(_driver.OnDetectEnabled)
+            {
+                _detectingMotion = true;
+                _driver?.ResetAnalyser();
+            }
+            else
+            {
+                _driver.OnDetectEnabled = true;
+            }
         }
 
         /// <inheritdoc />
-        public void DisableMotionDetection()
+        public void DisableMotionDetection(bool disableCallbackOnly = false)
         {
-            _detectingMotion = false;
+            if(disableCallbackOnly)
+            {
+                _driver.OnDetectEnabled = false;
+            }
+            else
+            {
+                _detectingMotion = false;
+            }
         }
 
         /// <inheritdoc />
@@ -158,5 +184,14 @@ private void WriteStreamToFile()
             this.MostRecentFilename = filename;
             this.MostRecentPathname = pathname;
         }
+
+        // This is used for temporary local-development performance testing. See the
+        // commentedlines before and inside FrameDiffDriver around the Apply method.
+        // public override void Dispose()
+        // {
+        //     long perf = (long)((float)_driver.totalElapsed / _driver.frameCounter);
+        //     Console.WriteLine($"{perf} ms/frame, total {_driver.frameCounter} frames");
+        //     base.Dispose();
+        // }
     }
 }

src/MMALSharp.Processing/Handlers/IMotionCaptureHandler.cs

@@ -33,6 +33,8 @@ public interface IMotionCaptureHandler
         /// <summary>
         /// Disables motion detection. When configured, this will instruct the capture handler not to detect motion.
         /// </summary>
-        void DisableMotionDetection();
+        /// <param name="disableCallbackOnly">When true, motion detection will continue but the OnDetect callback
+        /// will not be invoked. Call <see cref="EnableMotionDetection"/> to re-enable the callback.</param>
+        void DisableMotionDetection(bool disableCallbackOnly = false);
     }
 }

src/MMALSharp.Processing/Handlers/OutputCaptureHandler.cs

@@ -64,10 +64,10 @@ public virtual void PostProcess()
         }
 
         /// <summary>
-        /// Allows manipulating of the image frame.
+        /// Allows manipulation of the image frame.
         /// </summary>
         /// <param name="context">A delegate to the manipulation you wish to carry out.</param>
-        /// <param name="storeFormat">The image format to save manipulated files in..</param>
+        /// <param name="storeFormat">The image format to save manipulated files in, or null to return raw data.</param>
         public void Manipulate(Action<IFrameProcessingContext> context, ImageFormat storeFormat)
         {
             this.OnManipulate = context;

src/MMALSharp.Processing/MMALSharp.Processing.csproj

@@ -18,7 +18,8 @@
 	<VersionPrefix>0.7.0</VersionPrefix>
 	<CodeAnalysisRuleSet>..\..\StyleCop.Analyzers.ruleset</CodeAnalysisRuleSet>	
 	<DocumentationFile>bin\$(Configuration)\$(TargetFramework)\$(AssemblyName).xml</DocumentationFile>
-	<ProjectGuid>{dabc9991-56ad-4235-ba86-63def12c261a}</ProjectGuid> <!--Project guid for Sonar-->
+	<ProjectGuid>{dabc9991-56ad-4235-ba86-63def12c261a}</ProjectGuid>
+	<!--Project guid for Sonar-->
   </PropertyGroup>
 
   <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">