Skip to content
This repository has been archived by the owner on Feb 22, 2024. It is now read-only.

Motion detection plug-in model and analysis #175

Merged
merged 15 commits into from
Oct 12, 2020
Merged

Motion detection plug-in model and analysis #175

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
bcf3a21
Merge pull request #3 from techyian/dev
MV10 Aug 30, 2020
82627aa
Merge pull request #5 from techyian/dev
MV10 Aug 30, 2020
14298ad
motion algorithm plug-in model and live analysis
MV10 Sep 1, 2020
311147d
alter motion algorithm to output analysis frames to capture handler
MV10 Sep 8, 2020
391375f
remove auto pkg generation (don't break the build!)
MV10 Sep 8, 2020
c10fb33
image processing (convolution) using cell-based parallel processing
MV10 Sep 16, 2020
5fd1f0a
Merge branch 'dev' into motiondiff
MV10 Sep 16, 2020
883f0cb
restore fixes in commit 8b07ab3
MV10 Oct 6, 2020
20b591c
catch up to dev
MV10 Oct 6, 2020
b85084e
encoded image support
MV10 Oct 9, 2020
6ff0288
add cell counts for padded raw buffer dimensions
MV10 Oct 10, 2020
c18ece5
suppress callback if buffer length is zero
MV10 Oct 10, 2020
a2f47c7
fixes #187
MV10 Oct 11, 2020
7ca06e0
test that the source data is raw
MV10 Oct 11, 2020
62f7d02
copy cloned resolution into original context (TakePicture doesn't set…
MV10 Oct 11, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 15 additions & 13 deletions src/MMALSharp.Common/ImageContext.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,54 +13,56 @@ namespace MMALSharp.Common
/// </summary>
public class ImageContext
{
// Fields are used rather than properties for hot-path performance reasons.

/// <summary>
/// The working data.
/// </summary>
public byte[] Data { get; set; }
public byte[] Data;

/// <summary>
/// Flag to indicate whether image frame is raw.
/// </summary>
public bool Raw { get; set; }
public bool Raw;

/// <summary>
/// The resolution of the frame we're processing.
/// </summary>
public Resolution Resolution { get; set; }
public Resolution Resolution;

/// <summary>
/// The encoding format of the frame we're processing.
/// </summary>
public MMALEncoding Encoding { get; set; }
public MMALEncoding Encoding;

/// <summary>
/// The pixel format of the frame we're processing.
/// </summary>
public MMALEncoding PixelFormat { get; set; }
public MMALEncoding PixelFormat;

/// <summary>
/// The image format to store the processed data in.
/// </summary>
public ImageFormat StoreFormat { get; set; }
public ImageFormat StoreFormat;

/// <summary>
/// Indicates if this frame represents the end of the stream.
/// </summary>
public bool Eos { get; set; }
public bool Eos;

/// <summary>
/// Indicates if this frame contains IFrame data.
/// </summary>
public bool IFrame { get; set; }
public bool IFrame;

/// <summary>
/// The timestamp value.
/// </summary>
public long? Pts { get; set; }
public long? Pts;

/// <summary>
/// The pixel format stride.
/// </summary>
public int Stride { get; set; }
public int Stride;
}
}
3 changes: 2 additions & 1 deletion src/MMALSharp.Common/MMALSharp.Common.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@
<VersionPrefix>0.7.0</VersionPrefix>
<CodeAnalysisRuleSet>..\..\StyleCop.Analyzers.ruleset</CodeAnalysisRuleSet>
<DocumentationFile>bin\$(Configuration)\$(TargetFramework)\$(AssemblyName).xml</DocumentationFile>
<ProjectGuid>{65a1440e-72e1-4943-b469-5cfba8cb5633}</ProjectGuid> <!--Project guid for Sonar-->
<ProjectGuid>{65a1440e-72e1-4943-b469-5cfba8cb5633}</ProjectGuid>
<!--Project guid for Sonar-->
</PropertyGroup>
<PropertyGroup Condition="'$(Platform)'=='AnyCPU'">
<AllowUnsafeBlocks>True</AllowUnsafeBlocks>
Expand Down
2 changes: 1 addition & 1 deletion src/MMALSharp.FFmpeg/Handlers/FFmpegCaptureHandler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ public static ExternalProcessCaptureHandler RawVideoToMP4(string directory, stri
{
Filename = "ffmpeg",
Arguments = $"-framerate {fps} -i - -b:v {bitrate}k -c copy -movflags +frag_keyframe+separate_moof+omit_tfhd_offset+empty_moov {directory.TrimEnd()}/{filename}.mp4",
EchoOutput = true,
EchoOutput = echoOutput,
DrainOutputDelayMs = 500, // default
TerminationSignals = ExternalProcessCaptureHandlerOptions.SignalsFFmpeg
};
Expand Down
31 changes: 28 additions & 3 deletions src/MMALSharp.FFmpeg/Handlers/VLCCaptureHandler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,25 +15,50 @@ public static class VLCCaptureHandler
private static readonly string _VLCInternalMimeBoundaryName = "7b3cc56e5f51db803f790dad720ed50a";

/// <summary>
/// Listens for a request on the given port and begins streaming MJPEG images when a client connects.
/// Listens for a request on the given port and begins streaming MJPEG images when a client connects. Requires h.264 encoded I420 (YUV420p) as input.
/// </summary>
/// <param name="listenPort">The port to listen on. Defaults to 8554.</param>
/// <param name="echoOutput">Whether to echo stdout and stderr to the console or suppress it. Defaults to true.</param>
/// <param name="maxBitrate">Maximum output bitrate. If source data is available at a higher bitrate, VLC caps to this. Defaults to 2500 (25Mbps).</param>
/// <param name="maxFps">Maximum output framerate. If source data is available at a higher framerate, VLC caps to this. Defaults to 20.</param>
/// <returns>An initialized instance of <see cref="ExternalProcessCaptureHandler"/></returns>
public static ExternalProcessCaptureHandler StreamMJPEG(int listenPort = 8554, bool echoOutput = true, int maxBitrate = 2500, int maxFps = 20)
public static ExternalProcessCaptureHandler StreamH264asMJPEG(int listenPort = 8554, bool echoOutput = true, int maxBitrate = 2500, int maxFps = 20)
{
var opts = new ExternalProcessCaptureHandlerOptions
{
Filename = "cvlc",
Arguments = $"stream:///dev/stdin --sout \"#transcode{{vcodec=mjpg,vb={maxBitrate},fps={maxFps},acodec=none}}:standard{{access=http{{mime=multipart/x-mixed-replace;boundary=--{_VLCInternalMimeBoundaryName}}},mux=mpjpeg,dst=:{listenPort}/}}\" :demux=h264",
Arguments = $"stream:///dev/stdin --sout \"#transcode{{vcodec=mjpg,vb={maxBitrate},fps={maxFps},acodec=none}}:standard{{access=http{{mime=multipart/x-mixed-replace;boundary={_VLCInternalMimeBoundaryName}}},mux=mpjpeg,dst=:{listenPort}/}}\" :demux=h264",
EchoOutput = echoOutput,
DrainOutputDelayMs = 500, // default
TerminationSignals = ExternalProcessCaptureHandlerOptions.SignalsVLC
};

return new ExternalProcessCaptureHandler(opts);
}

/// <summary>
/// Listens for a request on the given port and begins streaming MJPEG images when a client connects. Requires raw RGB24 frames as input.
/// </summary>
/// <param name="width">The width of the raw frames. Defaults to 640.</param>
/// <param name="height">The height of the raw frames. Defaults to 480.</param>
/// <param name="fps">Expected FPS of the raw frames. Defaults to 24.</param>
/// <param name="listenPort">The port to listen on. Defaults to 8554.</param>
/// <param name="echoOutput">Whether to echo stdout and stderr to the console or suppress it. Defaults to true.</param>
/// <param name="maxBitrate">Maximum output bitrate. If source data is available at a higher bitrate, VLC caps to this. Defaults to 2500 (25Mbps).</param>
/// <param name="maxFps">Maximum output framerate. If source data is available at a higher framerate, VLC caps to this. Defaults to 20.</param>
/// <returns>An initialized instance of <see cref="ExternalProcessCaptureHandler"/></returns>
public static ExternalProcessCaptureHandler StreamRawRGB24asMJPEG(int width = 640, int height = 480, int fps = 24, int listenPort = 8554, bool echoOutput = true, int maxBitrate = 2500, int maxFps = 20)
{
var opts = new ExternalProcessCaptureHandlerOptions
{
Filename = "/bin/bash",
EchoOutput = true,
Arguments = $"-c \"ffmpeg -hide_banner -f rawvideo -c:v rawvideo -pix_fmt rgb24 -s:v {width}x{height} -r {fps} -i - -f h264 -c:v libx264 -preset ultrafast -tune zerolatency -vf format=yuv420p - | cvlc stream:///dev/stdin --sout '#transcode{{vcodec=mjpg,vb={maxBitrate},fps={maxFps},acodec=none}}:standard{{access=http{{mime=multipart/x-mixed-replace;boundary={_VLCInternalMimeBoundaryName}}},mux=mpjpeg,dst=:{listenPort}/}}' :demux=h264\"",
DrainOutputDelayMs = 500, // default = 500
TerminationSignals = ExternalProcessCaptureHandlerOptions.SignalsFFmpeg
};

return new ExternalProcessCaptureHandler(opts);
}
}
}
3 changes: 2 additions & 1 deletion src/MMALSharp.FFmpeg/MMALSharp.FFmpeg.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@
<VersionPrefix>0.7.0</VersionPrefix>
<CodeAnalysisRuleSet>..\..\StyleCop.Analyzers.ruleset</CodeAnalysisRuleSet>
<DocumentationFile>bin\$(Configuration)\$(TargetFramework)\$(AssemblyName).xml</DocumentationFile>
<ProjectGuid>{0600c674-e587-4267-89f3-b52ae9591f80}</ProjectGuid> <!--Project guid for Sonar-->
<ProjectGuid>{0600c674-e587-4267-89f3-b52ae9591f80}</ProjectGuid>
<!--Project guid for Sonar-->
</PropertyGroup>
<PropertyGroup Condition="'$(Platform)'=='AnyCPU'">
<AllowUnsafeBlocks>True</AllowUnsafeBlocks>
Expand Down
9 changes: 0 additions & 9 deletions src/MMALSharp.Processing/Handlers/ExternalProcessCaptureHandler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,15 +112,6 @@ public void PostProcess()
public string GetDirectory()
=> throw new NotImplementedException();

/// <summary>
/// Not used.
/// </summary>
/// <param name="allocSize">N/A.</param>
/// <returns>A NotImplementedException.</returns>
/// <exception cref="NotImplementedException"></exception>
public ProcessResult Process(uint allocSize)
=> throw new NotImplementedException();

/// <summary>
/// Writes frame data to the StandardInput stream for processing.
/// </summary>
Expand Down
53 changes: 44 additions & 9 deletions src/MMALSharp.Processing/Handlers/FrameBufferCaptureHandler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,8 @@ namespace MMALSharp.Handlers
/// </summary>
public class FrameBufferCaptureHandler : MemoryStreamCaptureHandler, IMotionCaptureHandler, IVideoCaptureHandler
{
private MotionConfig _motionConfig;
private bool _detectingMotion;
private FrameDiffAnalyser _motionAnalyser;
private FrameDiffDriver _driver;

private bool _waitForFullFrame = true;
private bool _writeFrameRequested = false;
Expand All @@ -45,6 +44,20 @@ public FrameBufferCaptureHandler()
: base()
{ }

/// <summary>
/// Creates a new <see cref="FrameBufferCaptureHandler"/> configured for motion detection analysis (either using a recorded
/// raw video stream where MMALStandalone.Instance is used, or when the camera is used but triggering motion detection events
/// is unnecessary). If motion detection events are desired, use the camera's WithMotionDetection method.
/// </summary>
/// <param name="motionConfig">The motion configuration.</param>
/// <param name="onDetect">A callback for when motion is detected.</param>
public FrameBufferCaptureHandler(MotionConfig motionConfig, Action onDetect)
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm assuming this additional constructor is used to replicate what MMALCamera.WithMotionDetection() does?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

By funny coincidence I was just looking at that myself, and I don't think it is needed. There was a point where I was trying to avoid modifying MotionConfig and I think it's left over from that. I will straighten it out.

Copy link
Collaborator Author

@MV10 MV10 Sep 15, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually that constructor was useful when doing nothing but visualizing the motion detection analysis. Used in a simple configuration like this:

using (var shell = new ExternalProcessCaptureHandler(raw_to_mjpeg_stream))
using (var motion = new FrameBufferCaptureHandler(motionConfig, null))
using (var resizer = new MMALIspComponent())
{
    motionAlgorithm.EnableAnalysis(shell);

    resizer.ConfigureOutputPort<VideoPort>(0, new MMALPortConfig(MMALEncoding.RGB24, MMALEncoding.RGB24, width: 640, height: 480), motion);
    cam.Camera.VideoPort.ConnectTo(resizer);

    await cameraWarmupDelay(cam);
    var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(seconds));
    await Task.WhenAll(new Task[]{
            shell.ProcessExternalAsync(timeout.Token),
            cam.ProcessAsync(cam.Camera.VideoPort, timeout.Token),
        }).ConfigureAwait(false);
}

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh I see I do explain one usage in the constructor summary, although it could be better, it does now also work with the camera (the example above is for streaming). I'll update the summary.

Creates a new configured for motion detection using a raw video stream where MMALStandalone.Instance is used (such as processing a pre-recorded file) rather than camera-based processing.

: base()
{
_driver = new FrameDiffDriver(motionConfig, onDetect);
_detectingMotion = true;
}

/// <summary>
/// Target directory when <see cref="WriteFrame"/> is invoked without a directory argument.
/// </summary>
Expand Down Expand Up @@ -99,7 +112,7 @@ public override void Process(ImageContext context)

if (_detectingMotion)
{
_motionAnalyser.Apply(context);
_driver.Apply(context);
}

// accumulate frame data in the underlying memory stream
Expand All @@ -122,22 +135,35 @@ public override void Process(ImageContext context)
/// <inheritdoc />
public void ConfigureMotionDetection(MotionConfig config, Action onDetect)
{
_motionConfig = config;
_motionAnalyser = new FrameDiffAnalyser(config, onDetect);
_driver = new FrameDiffDriver(config, onDetect);
this.EnableMotionDetection();
}

/// <inheritdoc />
public void EnableMotionDetection()
{
_detectingMotion = true;
_motionAnalyser?.ResetAnalyser();
if(_driver.OnDetectEnabled)
{
_detectingMotion = true;
_driver?.ResetAnalyser();
}
else
{
_driver.OnDetectEnabled = true;
}
}

/// <inheritdoc />
public void DisableMotionDetection()
public void DisableMotionDetection(bool disableCallbackOnly = false)
{
_detectingMotion = false;
if(disableCallbackOnly)
{
_driver.OnDetectEnabled = false;
}
else
{
_detectingMotion = false;
}
}

/// <inheritdoc />
Expand All @@ -158,5 +184,14 @@ private void WriteStreamToFile()
this.MostRecentFilename = filename;
this.MostRecentPathname = pathname;
}

// This is used for temporary local-development performance testing. See the
// commentedlines before and inside FrameDiffDriver around the Apply method.
// public override void Dispose()
// {
// long perf = (long)((float)_driver.totalElapsed / _driver.frameCounter);
// Console.WriteLine($"{perf} ms/frame, total {_driver.frameCounter} frames");
// base.Dispose();
// }
}
}
4 changes: 3 additions & 1 deletion src/MMALSharp.Processing/Handlers/IMotionCaptureHandler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ public interface IMotionCaptureHandler
/// <summary>
/// Disables motion detection. When configured, this will instruct the capture handler not to detect motion.
/// </summary>
void DisableMotionDetection();
/// <param name="disableCallbackOnly">When true, motion detection will continue but the OnDetect callback
/// will not be invoked. Call <see cref="EnableMotionDetection"/> to re-enable the callback.</param>
void DisableMotionDetection(bool disableCallbackOnly = false);
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If a user calls this method and wants to re-enable the callback, how do they do this? Am I right in thinking they just modify OnDetectEnabled against a FrameDiffDriver instance?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Calling EnableMotionDetection will re-enable the callback. I will add that to the parameter docs.

}
}
12 changes: 8 additions & 4 deletions src/MMALSharp.Processing/Handlers/InMemoryCaptureHandler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,21 +22,21 @@ public class InMemoryCaptureHandler : OutputCaptureHandler
/// The working data store.
/// </summary>
public List<byte> WorkingData { get; set; }

/// <summary>
/// Creates a new instance of <see cref="InMemoryCaptureHandler"/>.
/// </summary>
public InMemoryCaptureHandler()
{
this.WorkingData = new List<byte>();
}

/// <inheritdoc />
public override void Dispose()
{
MMALLog.Logger.LogInformation($"Successfully processed {Helpers.ConvertBytesToMegabytes(_totalProcessed)}.");
}

/// <inheritdoc />
public override void Process(ImageContext context)
{
Expand All @@ -51,7 +51,11 @@ public override void Process(ImageContext context)
/// </summary>
public override void PostProcess()
{
if (this.OnManipulate != null && this.ImageContext != null)
// When the context data length is zero, the data in the stream is a partial frame due to a race condition
// where the hardware has started the next frame before the library has begun the shutdown process. The buffer
// which triggered the call to PostProcess (from PortCallbackHandler) has a zero length which is what we're
// checking for here.
if (this.OnManipulate != null && this.ImageContext != null && this.ImageContext.Data.Length > 0)
{
this.ImageContext.Data = this.WorkingData.ToArray();
this.OnManipulate(new FrameProcessingContext(this.ImageContext));
Expand Down
4 changes: 2 additions & 2 deletions src/MMALSharp.Processing/Handlers/OutputCaptureHandler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,10 @@ public virtual void PostProcess()
}

/// <summary>
/// Allows manipulating of the image frame.
/// Allows manipulation of the image frame.
/// </summary>
/// <param name="context">A delegate to the manipulation you wish to carry out.</param>
/// <param name="storeFormat">The image format to save manipulated files in..</param>
/// <param name="storeFormat">The image format to save manipulated files in, or null to return raw data.</param>
public void Manipulate(Action<IFrameProcessingContext> context, ImageFormat storeFormat)
{
this.OnManipulate = context;
Expand Down
14 changes: 9 additions & 5 deletions src/MMALSharp.Processing/Handlers/StreamCaptureHandler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ public abstract class StreamCaptureHandler<T> : OutputCaptureHandler
/// A Stream instance that we can process image data to.
/// </summary>
public T CurrentStream { get; protected set; }

/// <inheritdoc />
public override void Process(ImageContext context)
{
Expand All @@ -44,7 +44,11 @@ public override void PostProcess()
{
if (this.CurrentStream != null && this.CurrentStream.CanRead && this.CurrentStream.Length > 0)
{
if (this.OnManipulate != null && this.ImageContext != null)
// When the context data length is zero, the data in the stream is a partial frame due to a race condition
// where the hardware has started the next frame before the library has begun the shutdown process. The buffer
// which triggered the call to PostProcess (from PortCallbackHandler) has a zero length which is what we're
// checking for here.
if (this.OnManipulate != null && this.ImageContext != null && this.ImageContext.Data.Length > 0)
{
byte[] arr = null;

Expand All @@ -64,11 +68,11 @@ public override void PostProcess()
}

using (var ms = new MemoryStream(this.ImageContext.Data))
{
{
this.CurrentStream.SetLength(0);
this.CurrentStream.Position = 0;
ms.CopyTo(this.CurrentStream);
}
}
}
}
}
Expand All @@ -77,7 +81,7 @@ public override void PostProcess()
MMALLog.Logger.LogWarning($"Something went wrong while processing stream: {e.Message}. {e.InnerException?.Message}. {e.StackTrace}");
}
}

/// <inheritdoc />
public override string TotalProcessed()
{
Expand Down
3 changes: 2 additions & 1 deletion src/MMALSharp.Processing/MMALSharp.Processing.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@
<VersionPrefix>0.7.0</VersionPrefix>
<CodeAnalysisRuleSet>..\..\StyleCop.Analyzers.ruleset</CodeAnalysisRuleSet>
<DocumentationFile>bin\$(Configuration)\$(TargetFramework)\$(AssemblyName).xml</DocumentationFile>
<ProjectGuid>{dabc9991-56ad-4235-ba86-63def12c261a}</ProjectGuid> <!--Project guid for Sonar-->
<ProjectGuid>{dabc9991-56ad-4235-ba86-63def12c261a}</ProjectGuid>
<!--Project guid for Sonar-->
</PropertyGroup>

<PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
Expand Down
Loading