Revise and refactor methods to do with Local Contrast Normalisation

Issue #370

src/AnalysisPrograms/Audio2InputForConvCNN.cs

@@ -666,7 +666,7 @@ public static AudioToSonogramResult GenerateSpectrogramImages(FileInfo sourceRec
 
             ////LoggedConsole.WriteLine("LCN: FramesPerSecond (Prior to LCN) = {0}", sonogram.FramesPerSecond);
             ////LoggedConsole.WriteLine("LCN: Neighbourhood of {0} seconds = {1} frames", neighbourhoodSeconds, neighbourhoodFrames);
-            sonogram.Data = NoiseRemoval_Briggs.NoiseReduction_ShortRecordings_SubtractAndLCN(sonogram.Data, lowPercentile, neighbourhoodFrames, lcnContrastLevel);
+            sonogram.Data = NoiseRemoval_Briggs.NoiseReduction_byLCN(sonogram.Data, lowPercentile, neighbourhoodFrames, lcnContrastLevel);
 
             // draw amplitude spectrogram unannotated
             FileInfo outputImage1 = new FileInfo(Path.Combine(outputDirectory.FullName, sourceName + ".amplitd.bmp"));

src/AnalysisPrograms/OscillationsGeneric.cs

@@ -184,7 +184,7 @@ public static void Main(Arguments arguments)
             // LocalContrastNormalisation over frequency bins is better and faster.
             int neighbourhood = 15;
             double contrastLevel = 0.5;
-            sonogram.Data = NoiseRemoval_Briggs.NoiseReduction_byLCNDivision(sonogram.Data, neighbourhood, contrastLevel);
+            sonogram.Data = NoiseRemoval_Briggs.NoiseReduction_byLCN(sonogram.Data, neighbourhood, contrastLevel);
 
             // ###############################################################
             // lowering the sensitivity threshold increases the number of hits.

src/AnalysisPrograms/SURFAnalysis.cs

@@ -385,7 +385,7 @@ public static AudioToSonogramResult GenerateSpectrogramImages(FileInfo sourceRec
 
             //LoggedConsole.WriteLine("LCN: FramesPerSecond (Prior to LCN) = {0}", sonogram.FramesPerSecond);
             //LoggedConsole.WriteLine("LCN: Neighbourhood of {0} seconds = {1} frames", neighbourhoodSeconds, neighbourhoodFrames);
-            sonogram.Data = NoiseRemoval_Briggs.NoiseReduction_ShortRecordings_SubtractAndLCN(sonogram.Data, lowPercentile, neighbourhoodFrames, lcnContrastLevel);
+            sonogram.Data = NoiseRemoval_Briggs.NoiseReduction_byLCN(sonogram.Data, lowPercentile, neighbourhoodFrames, lcnContrastLevel);
 
             // draw amplitude spectrogram unannotated
             FileInfo outputImage1 = new FileInfo(Path.Combine(opDir.FullName, sourceName + ".amplitd.png"));

src/AnalysisPrograms/SpectrogramGenerator/SpectrogramGenerator.Core.cs

@@ -555,11 +555,14 @@ public static Image<Rgb24> GetLcnSpectrogram(
             LoggedConsole.WriteLine("LCN: Neighbourhood of {0} seconds = {1} frames", neighbourhoodSeconds, neighbourhoodFrames);
 
             // subtract the lowest 20% of frames. This is first step in LCN noise removal. Sets the baseline.
-            const int lowPercentile = 20;
+            //const int lowPercentile = 20;
+            //spectrogram.Data =
+            //    NoiseRemoval_Briggs.NoiseReduction_byLowestPercentileSubtraction(spectrogram.Data, lowPercentile);
             spectrogram.Data =
-                NoiseRemoval_Briggs.NoiseReduction_byLowestPercentileSubtraction(spectrogram.Data, lowPercentile);
-            spectrogram.Data =
-                NoiseRemoval_Briggs.NoiseReduction_byLCNDivision(spectrogram.Data, neighbourhoodFrames, lcnContrastLevel);
+                NoiseRemoval_Briggs.NoiseReduction_byLCN(spectrogram.Data, neighbourhoodFrames, lcnContrastLevel);
+
+            //spectrogram.Data =
+            //            NoiseRemoval_Briggs.NoiseReduction_byLCNDivision(spectrogram.Data, lowPercent: 20, neighbourhoodFrames, lcnContrastLevel);
 
             // Finally background noise removal. This step is optional.
             double[] spectralDecibelBgn = NoiseProfile.CalculateBackgroundNoise(spectrogram.Data);

src/AudioAnalysisTools/Events/EventExtentions.cs

@@ -451,41 +451,5 @@ public static IEnumerable<ISpectralPoint> GetCompositeTrack<T>(IEnumerable<T> ev
 
             return maxAmplitudePoints.OrderBy(p => p);
         }
-
-        /// <summary>
-        /// Some events close to microphone. Reduces the signal amplitude.
-        /// </summary>
-        public static List<EventCommon> FilterEventsAfterAmplitudeNormalisation(
-            List<EventCommon> events,
-            double[,] spectrogramData,
-            UnitConverters converter,
-            double decibelThreshold)
-        {
-            // set maximum decibel value
-            var filteredEvents = new List<EventCommon>();
-
-            foreach (var ev in events)
-            {
-                /*
-                var decibelArray = GetDecibelArrayFromEvent(ev, spectrogramData, converter);
-
-                var tracks = GetCompositeTrack<IEnumerable<T>>(ev);
-
-                ev.Tracks;
-                var converter1 = ev.Converter;
-
-                // get the decibel profile
-                var decibelArray = GetDecibelArrayFromEvent(ev, spectrogramData, converter);
-                var maxDb = decibelArray.Max();
-
-                // adjust profile to new gain. First find the max decibel value in the event.
-                // if max decibels less than
-
-                filteredEvents.Add(ev);
-                */
-            }
-
-            return filteredEvents;
-        }
     }
 }

src/AudioAnalysisTools/Tracks/ForwardTrackAlgorithm.cs

@@ -109,20 +109,16 @@ public static (List<EventCommon> Events, double[] CombinedIntensity) GetForwardT
 
             List<EventCommon> returnEvents = events.Cast<EventCommon>().ToList();
 
-            // ########################## NEW WORK - NORMALISE AMPLITUDE AND THEN FILTER EVENTS Again.
-            //allResults.NewEvents = EventExtentions.FilterEventsUsingDynamicGain(
-            //    allResults.NewEvents.Cast<EventCommon>().ToList(),
-            //    allResults.Sonogram.Data,
-            //    decibelThreshold);
-            //Log.Debug($"Event count after adjusting gain = {allResults.NewEvents.Count}");
-
             // Combine coincident events that are stacked one above other.
             // This will help in some cases to combine related events.
             if (parameters.CombinePossibleHarmonics)
             {
                 returnEvents = CompositeEvent.CombineStackedEvents(events, parameters.HarmonicsStartDifference, parameters.HarmonicsHertzGap);
             }
 
+            // ##### Combining possible sequences currently happens at post-processing time - i.e. after all the profiles have been executed.
+            // ##### Combining possible sequences might be useful at the profile stage.
+            // So the following may be required in future - leave here for now.
             // Combine events that are temporally close and in the same frequency band.
             // This will help in some cases to combine related events.
             //if (parameters.CombinePossibleSyllableSequence)

src/TowseyLibrary/MatrixTools.cs

@@ -306,9 +306,9 @@ public static double[] GetRowAveragesOfSubmatrix(double[,] m, int r1, int c1, in
             return outM;
         }
 
-        /*
-         * converts a matrix to a vector by concatenating columns.
-         */
+        /// <summary>
+        /// Converts a matrix to a vector by concatenating its columns.
+        /// </summary>
         public static double[] Matrix2Array(double[,] matrix)
         {
             int ht = matrix.GetLength(0);

src/TowseyLibrary/NormalDist.cs

@@ -248,34 +248,6 @@ public static void AverageAndSD(double[,] data, out double av, out double sd)
             AverageAndSD(values, out av, out sd);
         }
 
-        /// <summary>
-        /// This was written for the local contrast normalisation (LCN) of amplitude spectrograms.
-        /// However the contrast is calculated wrt the local part of frequency bin or column.
-        /// Plugging up of ends of the returned array is a hack but it does not really matter.
-        /// </summary>
-        public static double[] CalculateLocalVariance(double[] data, int window)
-        {
-            int L = data.Length;
-            int halfwindow = window / 2;
-            double[] variances = new double[L];
-
-            for (int i = 0; i <= L - window; i++)
-            {
-                double[] subV = DataTools.Subarray(data, i, window);
-                AverageAndVariance(subV, out var av, out var variance);
-                variances[i + halfwindow] = variance;
-            }
-
-            // plug up the ends
-            for (int i = 0; i < halfwindow; i++)
-            {
-                variances[i] = variances[halfwindow];
-                variances[L - i - 1] = variances[L - halfwindow - 1];
-            }
-
-            return variances;
-        }
-
         public static double[] Normalise(double[] data)
         {
             AverageAndSD(data, out var av, out var sd);