Implement AudioBarVisualizer (#32)

* Implement AudioBarVisualizer

* Update livekit sdk to 2.9.0

* bump submodule client-sdk-android to v2.9.0

* spotless

* docs

.changeset/healthy-books-teach.md

@@ -0,0 +1,5 @@
+---
+"components-android": minor
+---
+
+Add AudioBarVisualizer for audio waveform visualizations

build.gradle

@@ -49,6 +49,9 @@ subprojects {
         }
         kotlin {
             target("src/*/java/**/*.kt")
+            targetExclude(
+                    "src/*/java/**/FFTAudioAnalyzer.kt", // Different license
+            )
             ktlint("0.50.0")
                     .setEditorConfigPath("$rootDir/.editorconfig")
             licenseHeaderFile(rootProject.file("LicenseHeaderFile.txt"))

livekit-compose-components/build.gradle

@@ -82,13 +82,14 @@ dokkaHtml {
     }
 }
 
-var livekitVersion = "2.8.1"
+var livekitVersion = "2.9.0"
 dependencies {
     api "io.livekit:livekit-android:${livekitVersion}"
 
     // For local development with the LiveKit Android SDK only.
     // api "io.livekit:livekit-android-sdk"
     implementation libs.kotlinx.serialization.json
+    implementation libs.noise
 
     implementation platform(libs.compose.bom)
     implementation 'androidx.compose.ui:ui'

livekit-compose-components/src/main/java/io/livekit/android/compose/ui/BarVisualizer.kt

@@ -0,0 +1,100 @@
+/*
+ * Copyright 2024 LiveKit, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.livekit.android.compose.ui
+
+import androidx.compose.animation.core.Spring
+import androidx.compose.animation.core.animateFloatAsState
+import androidx.compose.animation.core.spring
+import androidx.compose.foundation.Canvas
+import androidx.compose.foundation.layout.Box
+import androidx.compose.foundation.layout.fillMaxSize
+import androidx.compose.runtime.Composable
+import androidx.compose.ui.Modifier
+import androidx.compose.ui.geometry.CornerRadius
+import androidx.compose.ui.geometry.Offset
+import androidx.compose.ui.geometry.Size
+import androidx.compose.ui.graphics.Brush
+import androidx.compose.ui.graphics.Color
+import androidx.compose.ui.graphics.SolidColor
+import androidx.compose.ui.graphics.drawscope.DrawStyle
+import androidx.compose.ui.graphics.drawscope.Fill
+import androidx.compose.ui.unit.Dp
+import androidx.compose.ui.unit.dp
+import io.livekit.android.annotations.Beta
+
+private val springAnimation = spring<Float>(
+    stiffness = Spring.StiffnessHigh
+)
+
+/**
+ * Draws bars evenly split across the width of the composable.
+ *
+ * @param amplitudes Values of the bars, between 0.0f and 1.0f, where 1.0f represents the maximum height of the composable.
+ * @param alphas Alphas of the bars, between 0.0f and 1.0f. Defaults to 1.0f if null or not enough values are passed.
+ */
+@Beta
+@Composable
+fun BarVisualizer(
+    modifier: Modifier = Modifier,
+    style: DrawStyle = Fill,
+    brush: Brush = SolidColor(Color.Black),
+    barWidth: Dp = 8.dp,
+    minHeight: Float = 0.2f,
+    maxHeight: Float = 1.0f,
+    amplitudes: FloatArray,
+    alphas: FloatArray? = null,
+) {
+    val amplitudeStates = amplitudes.map { animateFloatAsState(targetValue = it, animationSpec = springAnimation) }
+    val alphaStates = alphas?.map { animateFloatAsState(targetValue = it) }
+    Box(
+        modifier = modifier
+    ) {
+        Canvas(
+            modifier = Modifier.fillMaxSize()
+        ) {
+            if (amplitudeStates.isEmpty()) {
+                return@Canvas
+            }
+            val barWidthPx = barWidth.toPx()
+            val innerSpacing = (size.width - barWidthPx * amplitudes.size) / (amplitudeStates.size - 1)
+            val barTotalWidth = barWidthPx + innerSpacing
+            amplitudeStates.forEachIndexed { index, amplitude ->
+                val normalizedAmplitude = minHeight + (maxHeight - minHeight) * amplitude.value.coerceIn(0.0f, 1.0f)
+                val alpha = if (alphaStates != null && index < alphaStates.size) {
+                    alphaStates[index].value
+                } else {
+                    1f
+                }
+
+                drawRoundRect(
+                    brush = brush,
+                    topLeft = Offset(
+                        x = index * barTotalWidth,
+                        y = size.height * (1 - normalizedAmplitude) / 2F
+                    ),
+                    size = Size(
+                        width = barWidthPx,
+                        height = (size.height * normalizedAmplitude).coerceAtLeast(1.dp.toPx())
+                    ),
+                    cornerRadius = CornerRadius(barWidthPx / 2, barWidthPx / 2),
+                    alpha = alpha,
+                    style = style
+                )
+            }
+        }
+    }
+}

livekit-compose-components/src/main/java/io/livekit/android/compose/ui/audio/AudioBarVisualizer.kt

@@ -0,0 +1,169 @@
+/*
+ * Copyright 2024 LiveKit, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.livekit.android.compose.ui.audio
+
+import androidx.compose.runtime.Composable
+import androidx.compose.runtime.DisposableEffect
+import androidx.compose.runtime.LaunchedEffect
+import androidx.compose.runtime.getValue
+import androidx.compose.runtime.mutableStateOf
+import androidx.compose.runtime.remember
+import androidx.compose.runtime.setValue
+import androidx.compose.ui.Modifier
+import androidx.compose.ui.graphics.Brush
+import androidx.compose.ui.graphics.Color
+import androidx.compose.ui.graphics.SolidColor
+import androidx.compose.ui.graphics.drawscope.DrawStyle
+import androidx.compose.ui.graphics.drawscope.Fill
+import io.livekit.android.annotations.Beta
+import io.livekit.android.compose.types.TrackReference
+import io.livekit.android.compose.ui.BarVisualizer
+import io.livekit.android.room.track.AudioTrack
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.launch
+import kotlin.math.pow
+import kotlin.math.round
+import kotlin.math.sqrt
+
+/**
+ * @param loPass the start index of the FFT samples to use (inclusive). 0 <= loPass < [hiPass].
+ * @param hiPass the end index of the FFT samples to use (exclusive). [loPass] < hiPass <= [FFTAudioAnalyzer.SAMPLE_SIZE].
+ */
+@Beta
+@Composable
+fun AudioBarVisualizer(
+    audioTrackRef: TrackReference?,
+    modifier: Modifier = Modifier,
+    barCount: Int = 15,
+    loPass: Int = 50,
+    hiPass: Int = 150,
+    style: DrawStyle = Fill,
+    brush: Brush = SolidColor(Color.Black),
+    alphas: FloatArray? = null,
+) {
+    val audioSink = remember(audioTrackRef) { AudioTrackSinkFlow() }
+    val audioProcessor = remember(audioTrackRef) { FFTAudioAnalyzer() }
+    val fftFlow = audioProcessor.fftFlow
+    var amplitudes by remember(audioTrackRef, barCount) { mutableStateOf(FloatArray(barCount)) }
+
+    // Attach the sink to the track.
+    DisposableEffect(key1 = audioTrackRef) {
+        val track = audioTrackRef?.publication?.track as? AudioTrack
+        track?.addSink(audioSink)
+
+        onDispose {
+            track?.removeSink(audioSink)
+            audioProcessor.release()
+        }
+    }
+
+    // Configure audio processor as needed.
+    LaunchedEffect(key1 = audioTrackRef) {
+        audioSink.audioFormat.collect {
+            audioProcessor.configure(it)
+        }
+    }
+
+    // Collect audio bytes and pass to audio fft analyzer
+    LaunchedEffect(key1 = audioTrackRef) {
+        launch(Dispatchers.IO) {
+            audioSink.audioFlow.collect { (buffer, _) ->
+                audioProcessor.queueInput(buffer)
+            }
+        }
+    }
+
+    // Process audio bytes into desired bars
+    LaunchedEffect(audioTrackRef, barCount) {
+        val averages = FloatArray(barCount)
+        launch(Dispatchers.IO) {
+            fftFlow.collect { fft ->
+                val sliced = fft.slice(loPass until hiPass)
+                amplitudes = calculateAmplitudeBarsFromFFT(sliced, averages, barCount)
+            }
+        }
+    }
+
+    BarVisualizer(
+        amplitudes = amplitudes,
+        modifier = modifier,
+        style = style,
+        brush = brush,
+        minHeight = 0.3f,
+        alphas = alphas,
+    )
+}
+
+private const val MIN_CONST = 2f
+private const val MAX_CONST = 25f
+
+private fun calculateAmplitudeBarsFromFFT(
+    fft: List<Float>,
+    averages: FloatArray,
+    barCount: Int,
+): FloatArray {
+    val amplitudes = FloatArray(barCount)
+    if (fft.isEmpty()) {
+        return amplitudes
+    }
+
+    // We average out the values over 3 occurences (plus the current one), so big jumps are smoothed out
+    // Iterate over the entire FFT result array.
+    for (barIndex in 0 until barCount) {
+        // Note: each FFT is a real and imaginary pair.
+        // Scale down by 2 and scale back up to ensure we get an even number.
+        val prevLimit = (round(fft.size.toFloat() / 2 * barIndex / barCount).toInt() * 2)
+            .coerceIn(0, fft.size - 1)
+        val nextLimit = (round(fft.size.toFloat() / 2 * (barIndex + 1) / barCount).toInt() * 2)
+            .coerceIn(0, fft.size - 1)
+
+        var accum = 0f
+        // Here we iterate within this single band
+        for (i in prevLimit until nextLimit step 2) {
+            // Convert real and imaginary part to get energy
+
+            val realSq = fft[i]
+                .toDouble()
+                .pow(2.0)
+            val imaginarySq = fft[i + 1]
+                .toDouble()
+                .pow(2.0)
+            val raw = sqrt(realSq + imaginarySq).toFloat()
+
+            accum += raw
+        }
+
+        // A window might be empty which would result in a 0 division
+        if ((nextLimit - prevLimit) != 0) {
+            accum /= (nextLimit - prevLimit)
+        } else {
+            accum = 0.0f
+        }
+
+        val smoothingFactor = 5
+        var avg = averages[barIndex]
+        avg += (accum - avg / smoothingFactor)
+        averages[barIndex] = avg
+
+        var amplitude = avg.coerceIn(MIN_CONST, MAX_CONST)
+        amplitude -= MIN_CONST
+        amplitude /= (MAX_CONST - MIN_CONST)
+        amplitudes[barIndex] = amplitude
+    }
+
+    return amplitudes
+}

livekit-compose-components/src/main/java/io/livekit/android/compose/ui/audio/AudioTrackSinkFlow.kt

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2024 LiveKit, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.livekit.android.compose.ui.audio
+
+import io.livekit.android.room.track.RemoteAudioTrack
+import kotlinx.coroutines.channels.BufferOverflow
+import kotlinx.coroutines.flow.MutableSharedFlow
+import kotlinx.coroutines.flow.MutableStateFlow
+import livekit.org.webrtc.AudioTrackSink
+import java.nio.ByteBuffer
+
+/**
+ * Gathers the audio data from a [RemoteAudioTrack] and emits through a flow.
+ */
+class AudioTrackSinkFlow : AudioTrackSink {
+    val audioFormat = MutableStateFlow(AudioFormat(16, 48000, 1))
+    val audioFlow = MutableSharedFlow<Pair<ByteBuffer, Int>>(
+        extraBufferCapacity = 1,
+        onBufferOverflow = BufferOverflow.DROP_OLDEST
+    )
+
+    override fun onData(
+        audioData: ByteBuffer,
+        bitsPerSample: Int,
+        sampleRate: Int,
+        numberOfChannels: Int,
+        numberOfFrames: Int,
+        absoluteCaptureTimestampMs: Long
+    ) {
+        val curAudioFormat = audioFormat.value
+        if (curAudioFormat.bitsPerSample != bitsPerSample ||
+            curAudioFormat.sampleRate != sampleRate ||
+            curAudioFormat.numberOfChannels != numberOfChannels
+        ) {
+            audioFormat.tryEmit(AudioFormat(bitsPerSample, sampleRate, numberOfChannels))
+        }
+        audioFlow.tryEmit(audioData to numberOfFrames)
+    }
+}
+
+data class AudioFormat(val bitsPerSample: Int, val sampleRate: Int, val numberOfChannels: Int)