From 0d143de8e34724b003f69a920d00c4f0b8fe44fa Mon Sep 17 00:00:00 2001 From: towsey Date: Fri, 16 Oct 2020 21:31:19 +1100 Subject: [PATCH] Refactor the sideband filter for acoustic events Issue #390 --- .../Recognizers/GenericRecognizer.cs | 37 +-- src/AudioAnalysisTools/Events/EventFilters.cs | 259 ++++-------------- 2 files changed, 66 insertions(+), 230 deletions(-) diff --git a/src/AnalysisPrograms/Recognizers/GenericRecognizer.cs b/src/AnalysisPrograms/Recognizers/GenericRecognizer.cs index 14e2a27b0..8a9b7b4ec 100644 --- a/src/AnalysisPrograms/Recognizers/GenericRecognizer.cs +++ b/src/AnalysisPrograms/Recognizers/GenericRecognizer.cs @@ -16,7 +16,6 @@ namespace AnalysisPrograms.Recognizers using AudioAnalysisTools; using AudioAnalysisTools.DSP; using AudioAnalysisTools.Events; - using AudioAnalysisTools.Events.Tracks; using AudioAnalysisTools.Events.Types; using AudioAnalysisTools.Indices; using AudioAnalysisTools.StandardSpectrograms; @@ -34,7 +33,7 @@ public class GenericRecognizer : RecognizerBase { private static readonly ILog Log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); - private bool combineOverlappedEvents = false; + private readonly bool combineOverlappedEvents = false; /// public override string Author => "Ecosounds"; @@ -358,31 +357,15 @@ public override RecognizerResults Recognize( var sidebandActivity = postprocessingConfig.SidebandActivity; if (sidebandActivity != null) { - if (sidebandActivity.LowerHertzBuffer > 0) - { - var spectralEvents2 = allResults.NewEvents.Cast().ToList(); - allResults.NewEvents = EventFilters.FilterEventsOnLowerSidebandActivity( - spectralEvents2, - allResults.Sonogram, - sidebandActivity.LowerHertzBuffer, - segmentStartOffset, - sidebandActivity.DecibelBuffer); - - Log.Debug($"Event count after filtering on acoustic activity in lower sideband = {allResults.NewEvents.Count}"); - } - - if (sidebandActivity.UpperHertzBuffer > 0) - { - var spectralEvents3 = allResults.NewEvents.Cast().ToList(); - allResults.NewEvents = EventFilters.FilterEventsOnUpperSidebandActivity( - spectralEvents3, - allResults.Sonogram, - sidebandActivity.UpperHertzBuffer, - segmentStartOffset, - sidebandActivity.DecibelBuffer); - - Log.Debug($"Event count after filtering on acoustic activity in upper sideband = {allResults.NewEvents.Count}"); - } + var spectralEvents2 = allResults.NewEvents.Cast().ToList(); + allResults.NewEvents = EventFilters.FilterEventsOnSidebandActivity( + spectralEvents2, + allResults.Sonogram, + sidebandActivity.LowerHertzBuffer, + sidebandActivity.UpperHertzBuffer, + sidebandActivity.DecibelBuffer, + segmentStartOffset); + Log.Debug($"Event count after filtering on acoustic activity in sidebands = {allResults.NewEvents.Count}"); } // Write out the events to log. diff --git a/src/AudioAnalysisTools/Events/EventFilters.cs b/src/AudioAnalysisTools/Events/EventFilters.cs index 7b964cef1..400a865df 100644 --- a/src/AudioAnalysisTools/Events/EventFilters.cs +++ b/src/AudioAnalysisTools/Events/EventFilters.cs @@ -9,7 +9,6 @@ namespace AudioAnalysisTools.Events using System.Linq; using AudioAnalysisTools.Events.Types; using AudioAnalysisTools.StandardSpectrograms; - using MoreLinq; using TowseyLibrary; public static class EventFilters @@ -267,7 +266,6 @@ public static (bool[] TemporalFootprint, double TimeScale) GetTemporalFootprint( foreach (var pair in startEnds) { - int startFrame = (int)Math.Floor((pair[0] - firstStart) / timeScale); int endFrame = startFrame - 1 + (int)Math.Floor(pair[1] / timeScale); @@ -289,70 +287,28 @@ public static (int Count, double AveragePeriod, double SdPeriod) GetPeriodicity( } /// - /// Removes events from a list of events that contain excessive noise in the upper neighbourhood. + /// Removes events from a list of events that contain excessive noise in the lower and/or upper neighbourhood. /// Excess noise can indicate that this is not a legitimate event. /// This method measures noise as the average decibel value in the buffer zones above and below the events. /// /// A list of spectral events. - /// A matrix of the spectrogram in which event occurs. + /// A matrix of the spectrogram in which event occurs. /// The band width of the required lower buffer. 100-200Hz is often appropriate. /// The band width of the required upper buffer. 300-500Hz is often appropriate. - /// Converts sec/Hz to frame/bin. - /// Threshold noise level - assumed to be in decibels. + /// The decibel threshold for acoustic activity in a sideband. + /// Start time of the current recording segment. /// A list of filtered events. - public static List FilterEventsOnNeighbourhoodAverage( - List events, - double[,] spectrogramData, - double lowerHertzBuffer, - double upperHertzBuffer, - UnitConverters converter, - double decibelThreshold) - { - // allow bin gaps above and below the event. - int upperBinGap = 4; - int lowerBinGap = 2; - - var filteredEvents = new List(); - foreach (var ev in events) - { - var avLowerNhAmplitude = GetAverageAmplitudeInLowerNeighbourhood((SpectralEvent)ev, spectrogramData, lowerHertzBuffer, lowerBinGap, converter); - var avUpperNhAmplitude = GetAverageAmplitudeInUpperNeighbourhood((SpectralEvent)ev, spectrogramData, upperHertzBuffer, upperBinGap, converter); - - // Require that both the lower and upper buffer zones contain less acoustic activity than the threshold. - if (avLowerNhAmplitude < decibelThreshold && avUpperNhAmplitude < decibelThreshold) - { - // There is little acoustic activity in the designated buffer zones. It is likely to be a discrete event. - filteredEvents.Add(ev); - } - } - - return filteredEvents; - } - - /// - /// Removes events from a list of events that contain excessive noise in the lower event side band. - /// Excess noise can indicate that this is not a legitimate event. - /// This method counts the bins and frames containing above threshold activity (decibel value) in the buffer zone below the events. - /// - /// A list of spectral events. - /// The decibel spectrogram in which the events occurs. - /// The band width of the required lower buffer. 100-200Hz is often appropriate. - /// Minimum required decibel difference between event activity and neighbourhood activity. - /// A list of filtered events. - public static List FilterEventsOnLowerSidebandActivity( + public static List FilterEventsOnSidebandActivity( List events, BaseSonogram spectrogram, int lowerHertzBuffer, - TimeSpan segmentStartOffset, - double decibelBuffer) + int upperHertzBuffer, + double decibelThreshold, + TimeSpan segmentStartOffset) { - if (lowerHertzBuffer == 0) - { - return null; - } - // allow bin gaps below the event. int lowerBinGap = 2; + int upperBinGap = 2; var converter = new UnitConverters( segmentStartOffset: segmentStartOffset.TotalSeconds, @@ -360,77 +316,67 @@ public static List FilterEventsOnLowerSidebandActivity( frameSize: spectrogram.Configuration.WindowSize, frameOverlap: spectrogram.Configuration.WindowOverlap); - var filteredEvents = new List(); + var spectrogramData = spectrogram.Data; + + var filteredEvents = new List(); foreach (var ev in events) { - var eventDecibels = EventExtentions.GetAverageDecibelsInEvent(ev, spectrogram.Data, converter); - var sidebandThreshold = eventDecibels - decibelBuffer; + var avEventDecibels = EventExtentions.GetAverageDecibelsInEvent(ev, spectrogramData, converter); + var retainEvent1 = true; + var retainEvent2 = true; - var sidebandMatrix = GetLowerNeighbourhood(ev, spectrogram.Data, lowerHertzBuffer, lowerBinGap, converter); + if (lowerHertzBuffer > 0) + { + var lowerSidebandMatrix = GetLowerEventSideband(ev, spectrogramData, lowerHertzBuffer, lowerBinGap, converter); + retainEvent1 = IsSidebandActivityBelowThreshold( + avEventDecibels, + lowerSidebandMatrix, + decibelThreshold); + } - var averageRowDecibels = MatrixTools.GetRowAverages(sidebandMatrix); - var averageColDecibels = MatrixTools.GetColumnAverages(sidebandMatrix); - int noisyRowCount = averageRowDecibels.Count(x => x > sidebandThreshold); - int noisyColCount = averageColDecibels.Count(x => x > sidebandThreshold); + if (upperHertzBuffer > 0) + { + var upperSidebandMatrix = GetUpperEventSideband(ev, spectrogramData, upperHertzBuffer, upperBinGap, converter); + retainEvent2 = IsSidebandActivityBelowThreshold( + avEventDecibels, + upperSidebandMatrix, + decibelThreshold); + } - // Require that there be at most one buffer bin and one buffer frame containing excessive acoustic activity. - if (noisyRowCount <= 1 && noisyColCount <= 1) + if (retainEvent1 && retainEvent2) { - // There is reduced acoustic activity in the upper and lower buffer zones. It is likely to be a discrete event. + // The acoustic activity in event sidebands is below the threshold. It is likely to be a discrete event. filteredEvents.Add(ev); } } - return filteredEvents; + var eventsCommon = filteredEvents.Cast().ToList(); + return eventsCommon; } - /// - /// Removes events from a list of events that contain excessive noise in the upper event side band. - /// Excess noise can indicate that this is not a legitimate event. - /// This method counts the bins and frames containing above threshold activity (decibel value) in the buffer zone above the events. - /// - /// A list of spectral events. - /// The decibel spectrogram in which the events occurs. - /// The band width of the required upper buffer. 300-500Hz is often appropriate. - /// Minimum required decibel difference between event activity and neighbourhood activity. - /// A list of filtered events. - public static List FilterEventsOnUpperSidebandActivity( - List events, - BaseSonogram spectrogram, - int upperHertzBuffer, - TimeSpan segmentStartOffset, - double decibelBuffer) + public static bool IsSidebandActivityBelowThreshold( + double avEventDecibels, + double[,] sidebandMatrix, + double sidebandThreshold) { - // allow bin gaps above the event. - int upperBinGap = 3; - - var converter = new UnitConverters( - segmentStartOffset: segmentStartOffset.TotalSeconds, - sampleRate: spectrogram.SampleRate, - frameSize: spectrogram.Configuration.WindowSize, - frameOverlap: spectrogram.Configuration.WindowOverlap); + var averageRowDecibels = MatrixTools.GetRowAverages(sidebandMatrix); + var averageColDecibels = MatrixTools.GetColumnAverages(sidebandMatrix); + var averageMatrixDecibels = averageColDecibels.Average(); - var filteredEvents = new List(); - foreach (var ev in events) + // Is the average acoustic activity in the sideband below the user set threshold? + bool avBgBelowThreshold = averageMatrixDecibels < sidebandThreshold; + if (!avBgBelowThreshold) { - var eventDecibels = EventExtentions.GetAverageDecibelsInEvent(ev, spectrogram.Data, converter); - var sidebandThreshold = eventDecibels - decibelBuffer; - - var sidebandMatrix = GetUpperNeighbourhood(ev, spectrogram.Data, upperHertzBuffer, upperBinGap, converter); - var averageRowDecibels = MatrixTools.GetRowAverages(sidebandMatrix); - var averageColDecibels = MatrixTools.GetColumnAverages(sidebandMatrix); - int noisyRowCount = averageRowDecibels.Count(x => x > sidebandThreshold); - int noisyColCount = averageColDecibels.Count(x => x > sidebandThreshold); - - // Require that there be at most one buffer bin and one buffer frame containing excessive acoustic activity. - if (noisyRowCount <= 1 && noisyColCount <= 1) - { - // There is reduced acoustic activity in the upper and lower buffer zones. It is likely to be a discrete event. - filteredEvents.Add(ev); - } + return false; } - return filteredEvents; + // Also need to cover possibility that there is much acoustic activity concentrated in one freq bin or time frame. + // Therefore, also require that there be at most one sideband bin and one sideband frame containing acoustic activity + // that is greater than the average in the event. + int noisyRowCount = averageRowDecibels.Count(x => x > avEventDecibels); + int noisyColCount = averageColDecibels.Count(x => x > avEventDecibels); + bool doRetain = noisyRowCount <= 1 && noisyColCount <= 1; + return doRetain; } /// @@ -440,8 +386,8 @@ public static List FilterEventsOnUpperSidebandActivity( /// The spectrogram data as matrix with origin top/left. /// THe bandwidth of the buffer zone in Hertz. /// A converter to convert seconds/Hertz to frames/bins. - /// The neighbourhood as a matrix. - public static double[,] GetLowerNeighbourhood(SpectralEvent ev, double[,] spectrogramData, double bufferHertz, int gap, UnitConverters converter) + /// The sideband as a matrix. + public static double[,] GetLowerEventSideband(SpectralEvent ev, double[,] spectrogramData, double bufferHertz, int gap, UnitConverters converter) { var bufferBins = (int)Math.Round(bufferHertz / converter.HertzPerFreqBin); var topBufferBin = converter.GetFreqBinFromHertz(ev.LowFrequencyHertz) - gap; @@ -463,7 +409,7 @@ public static List FilterEventsOnUpperSidebandActivity( /// The bandwidth of the buffer zone in Hertz. /// A converter to convert seconds/Hertz to frames/bins. /// The neighbourhood as a matrix. - public static double[,] GetUpperNeighbourhood(SpectralEvent ev, double[,] spectrogramData, double bufferHertz, int gap, UnitConverters converter) + public static double[,] GetUpperEventSideband(SpectralEvent ev, double[,] spectrogramData, double bufferHertz, int gap, UnitConverters converter) { var bufferBins = (int)Math.Round(bufferHertz / converter.HertzPerFreqBin); var bottomBufferBin = converter.GetFreqBinFromHertz(ev.HighFrequencyHertz) + gap; @@ -475,98 +421,5 @@ public static List FilterEventsOnUpperSidebandActivity( var subMatrix = MatrixTools.Submatrix(spectrogramData, frameStart, bottomBufferBin, frameEnd, topBufferBin); return subMatrix; } - - /// - /// Gets the upper and lower buffer zones (above and below an event). - /// Returns them as one combined matrix. - /// This makes it easier to determine the presense of acoustic events (especially wind) in the buffer zones. - /// - /// The event. - /// The spectrogram data as matrix with origin top/left. - /// The bandwidth of the lower buffer zone in Hertz. - /// Number of freq bins left as gap below event. - /// The bandwidth of the upper buffer zone in Hertz. - /// Number of freq bins left as gap above event. - /// A converter to convert seconds/Hertz to frames/bins. - /// A single matrix. - public static double[,] GetNeighbourhoodAsOneMatrix( - SpectralEvent ev, - double[,] spectrogramData, - double lowerHertzBuffer, - int lowerBinGap, - double upperHertzBuffer, - int upperBinGap, - UnitConverters converter) - { - double[,] subMatrix1 = null; - if (upperHertzBuffer > 0) - { - subMatrix1 = GetUpperNeighbourhood(ev, spectrogramData, upperHertzBuffer, upperBinGap, converter); - } - - double[,] subMatrix2 = null; - if (lowerHertzBuffer > 0) - { - subMatrix2 = GetLowerNeighbourhood(ev, spectrogramData, lowerHertzBuffer, lowerBinGap, converter); - } - - if (subMatrix1 == null && subMatrix2 == null) - { - return null; - } - - if (subMatrix1 == null) - { - return subMatrix2; - } - - if (subMatrix2 == null) - { - return subMatrix1; - } - - var matrix = MatrixTools.ConcatenateTwoMatrices(subMatrix1, subMatrix2); - return matrix; - } - - /// - /// Calculates the average amplitude in the frequency bins just above the event. - /// If it contains above threshold acoustic content, this is unlikely to be a discrete event. - /// NOTE: This method takes a simple average of log values. This is good enough for the purpose, although not mathematically correct. - /// Logs are computationally expensive. - /// - /// The event. - /// The spectrogram data as matrix with origin top/left. - /// THe bandwidth of the buffer zone in Hertz. - /// Number of freq bins as gap between event and buffer zone. - /// A converter to convert seconds/Hertz to frames/bins. - /// Unweighted average of the spectrogram amplitude in buffer band above the event. - public static double GetAverageAmplitudeInUpperNeighbourhood(SpectralEvent ev, double[,] spectrogramData, double bufferHertz, int binGap, UnitConverters converter) - { - var subMatrix = GetUpperNeighbourhood(ev, spectrogramData, bufferHertz, binGap, converter); - var averageRowDecibels = MatrixTools.GetRowAverages(subMatrix); - var av = averageRowDecibels.Average(); - return av; - } - - /// - /// Calculates the average amplitude in the frequency bins just below the event. - /// If it contains above threshold acoustic content, this is unlikely to be a discrete event. - /// NOTE: This method takes a simple average of log values. This is good enough for the purpose, although not mathematically correct. - /// Logs are computationally expensive. - /// - /// The event. - /// The spectrogram data as matrix with origin top/left. - /// The bandwidth of the buffer zone in bins. - /// Number of freq bins as gap between event and buffer zone. - /// A converter to convert seconds/Hertz to frames/bins. - /// Unweighted average of the spectrogram amplitude in buffer band below the event. - public static double GetAverageAmplitudeInLowerNeighbourhood(SpectralEvent ev, double[,] spectrogramData, double bufferHertz, int binGap, UnitConverters converter) - { - var subMatrix = GetLowerNeighbourhood(ev, spectrogramData, bufferHertz, binGap, converter); - var averageRowDecibels = MatrixTools.GetRowAverages(subMatrix); - var av = averageRowDecibels.Average(); - return av; - } } }