Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Capture Latency Estiamte Indicator for Extension #601

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add Capture Latency Estiamte Indicator for Extension #601

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions CameraXExtensions/app/src/main/java/com/example/android/cameraxextensions/MainActivity.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import android.util.Log
import androidx.activity.OnBackPressedCallback
import androidx.activity.result.contract.ActivityResultContracts
import androidx.appcompat.app.AppCompatActivity
import androidx.camera.core.ImageCaptureLatencyEstimate
import androidx.camera.extensions.ExtensionMode
import androidx.core.app.ActivityCompat
import androidx.lifecycle.Lifecycle
Expand Down Expand Up @@ -53,6 +54,11 @@ import kotlinx.coroutines.launch
* button will capture a photo and display the photo.
*/
class MainActivity : AppCompatActivity() {
private companion object {
const val TAG = "MainActivity"
const val CAPTURE_LATENCY_INDICATOR_THRESHOLD_MS = 500
}

private val extensionName = mapOf(
ExtensionMode.AUTO to R.string.camera_mode_auto,
ExtensionMode.NIGHT to R.string.camera_mode_night,
Expand Down Expand Up @@ -308,6 +314,7 @@ class MainActivity : AppCompatActivity() {
cameraExtensionsViewModel.initializeCamera()
}
CameraState.READY -> {
Log.d(TAG, "Camera is ready")
cameraExtensionsScreen.previewView.doOnLaidOut {
cameraExtensionsViewModel.startPreview(
this@MainActivity as LifecycleOwner,
Expand All @@ -330,6 +337,22 @@ class MainActivity : AppCompatActivity() {
}
)
}
CameraState.PREVIEW_ACTIVE -> {
Log.d(TAG, "Camera preview is active")
captureScreenViewState.emit(captureScreenViewState.value.updateCameraScreen { s ->
val latencyEstimate = cameraUiState.realtimeCaptureLatencyEstimate
if (latencyEstimate == ImageCaptureLatencyEstimate.UNDEFINED_IMAGE_CAPTURE_LATENCY) {
Log.d(TAG, "Camera preview is active: hide latency estimate indicator")
s.hideLatencyEstimateIndicator()
} else if (latencyEstimate.captureLatencyMillis <= CAPTURE_LATENCY_INDICATOR_THRESHOLD_MS) {
Log.d(TAG, "Camera preview is active: hide latency estimate indicator (under ${CAPTURE_LATENCY_INDICATOR_THRESHOLD_MS}ms)")
s.hideLatencyEstimateIndicator()
} else {
Log.d(TAG, "Camera preview is active: show latency estimate indicator (${latencyEstimate.captureLatencyMillis}ms")
s.showLatencyEstimateIndicator(latencyEstimate.captureLatencyMillis)
}
})
}
CameraState.PREVIEW_STOPPED -> Unit
}
}
Expand Down
9 changes: 9 additions & 0 deletions ...Extensions/app/src/main/java/com/example/android/cameraxextensions/model/CameraUiState.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import android.graphics.Bitmap
import androidx.camera.core.CameraSelector.LENS_FACING_BACK
import androidx.camera.core.CameraSelector.LensFacing
import androidx.camera.core.ImageCapture
import androidx.camera.core.ImageCaptureLatencyEstimate
import androidx.camera.extensions.ExtensionMode

/**
Expand All @@ -33,6 +34,8 @@ data class CameraUiState(
val availableCameraLens: List<Int> = listOf(LENS_FACING_BACK),
@LensFacing val cameraLens: Int = LENS_FACING_BACK,
@ExtensionMode.Mode val extensionMode: Int = ExtensionMode.NONE,
val realtimeCaptureLatencyEstimate: ImageCaptureLatencyEstimate =
ImageCaptureLatencyEstimate.UNDEFINED_IMAGE_CAPTURE_LATENCY
)

/**
Expand All @@ -49,6 +52,12 @@ enum class CameraState {
*/
READY,

/**
* Camera is open and preview stream is currently running.
* Updates during this period are from camera state operations.
*/
PREVIEW_ACTIVE,

/**
* Camera is initialized but the preview has been stopped.
*/
Expand Down
81 changes: 81 additions & 0 deletions ...ions/app/src/main/java/com/example/android/cameraxextensions/ui/CameraExtensionsScreen.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,9 @@ import com.google.android.material.progressindicator.CircularProgressIndicator
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.MutableSharedFlow
import kotlinx.coroutines.launch
import java.util.concurrent.TimeUnit
import kotlin.math.max
import kotlin.math.roundToInt

/**
* Displays the camera preview and captured photo.
Expand Down Expand Up @@ -90,6 +92,8 @@ class CameraExtensionsScreen(private val root: View) {
root.findViewById(R.id.processProgressContainer)
private val processProgressIndicator: CircularProgressIndicator =
root.findViewById(R.id.processProgressIndicator)
private val latencyEstimateIndicator: TextView =
root.findViewById(R.id.latencyEstimateIndicator)

val previewView: PreviewView = root.findViewById(R.id.previewView)

Expand Down Expand Up @@ -264,6 +268,77 @@ class CameraExtensionsScreen(private val root: View) {
processProgressIndicator.progress = 0
}

private fun showLatencyEstimate(latencyEstimateMillis: Long) {
val estimateSeconds = (latencyEstimateMillis.toFloat() / 1000).roundToInt()

if (!latencyEstimateIndicator.isVisible) {
val alphaAnimation =
SpringAnimation(latencyEstimateIndicator, DynamicAnimation.ALPHA, 1f).apply {
spring.stiffness = SpringForce.STIFFNESS_LOW
spring.dampingRatio = SpringForce.DAMPING_RATIO_NO_BOUNCY
}

val scaleAnimationX =
SpringAnimation(latencyEstimateIndicator, DynamicAnimation.SCALE_X, 1f).apply {
spring.stiffness = SpringForce.STIFFNESS_LOW
spring.dampingRatio = SpringForce.DAMPING_RATIO_LOW_BOUNCY
}

val scaleAnimationY =
SpringAnimation(latencyEstimateIndicator, DynamicAnimation.SCALE_Y, 1f).apply {
spring.stiffness = SpringForce.STIFFNESS_LOW
spring.dampingRatio = SpringForce.DAMPING_RATIO_LOW_BOUNCY
}

latencyEstimateIndicator.apply {
isVisible = true
alpha = 0f
scaleX = 0.2f
scaleY = 0.2f
}

alphaAnimation.start()
scaleAnimationX.start()
scaleAnimationY.start()
}

latencyEstimateIndicator.text =
context.getString(R.string.latency_estimate, estimateSeconds)
}

private fun hideLatencyEstimate() {
if (latencyEstimateIndicator.isVisible) {
val alphaAnimation =
SpringAnimation(latencyEstimateIndicator, DynamicAnimation.ALPHA, 0f).apply {
spring.stiffness = SpringForce.STIFFNESS_LOW
spring.dampingRatio = SpringForce.DAMPING_RATIO_NO_BOUNCY

addEndListener { _, canceled, _, _ ->
if (!canceled) {
latencyEstimateIndicator.isVisible = false
latencyEstimateIndicator.text = ""
}
}
}

val scaleAnimationX =
SpringAnimation(latencyEstimateIndicator, DynamicAnimation.SCALE_X, 0f).apply {
spring.stiffness = SpringForce.STIFFNESS_LOW
spring.dampingRatio = SpringForce.DAMPING_RATIO_LOW_BOUNCY
}

val scaleAnimationY =
SpringAnimation(latencyEstimateIndicator, DynamicAnimation.SCALE_Y, 0f).apply {
spring.stiffness = SpringForce.STIFFNESS_LOW
spring.dampingRatio = SpringForce.DAMPING_RATIO_LOW_BOUNCY
}

alphaAnimation.start()
scaleAnimationX.start()
scaleAnimationY.start()
}
}

private fun showPhoto(uri: Uri?) {
if (uri == null) return
photoPreview.isVisible = true
Expand Down Expand Up @@ -300,6 +375,12 @@ class CameraExtensionsScreen(private val root: View) {
} else {
hideProcessProgressIndicator()
}

if (state.latencyEstimateIndicatorViewState.isVisible) {
showLatencyEstimate(state.latencyEstimateIndicatorViewState.latencyEstimateMillis)
} else {
hideLatencyEstimate()
}
}

private fun onItemClick(view: View) {
Expand Down
85 changes: 81 additions & 4 deletions ...rc/main/java/com/example/android/cameraxextensions/viewmodel/CameraExtensionsViewModel.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,15 @@ package com.example.android.cameraxextensions.viewmodel

import android.app.Application
import android.graphics.Bitmap
import android.util.Log
import androidx.camera.core.AspectRatio
import androidx.camera.core.Camera
import androidx.camera.core.CameraSelector
import androidx.camera.core.CameraSelector.LensFacing
import androidx.camera.core.FocusMeteringAction
import androidx.camera.core.ImageCapture
import androidx.camera.core.ImageCaptureException
import androidx.camera.core.ImageCaptureLatencyEstimate
import androidx.camera.core.MeteringPoint
import androidx.camera.core.Preview
import androidx.camera.core.UseCaseGroup
Expand All @@ -41,11 +43,16 @@ import com.example.android.cameraxextensions.model.CameraUiState
import com.example.android.cameraxextensions.model.CaptureState
import com.example.android.cameraxextensions.repository.ImageCaptureRepository
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.asExecutor
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.guava.await
import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch
import androidx.lifecycle.asFlow
import kotlinx.coroutines.CoroutineScope

/**
* View model for camera extensions. This manages all the operations on the camera.
Expand All @@ -61,6 +68,11 @@ class CameraExtensionsViewModel(
private val application: Application,
private val imageCaptureRepository: ImageCaptureRepository
) : ViewModel() {
private companion object {
const val TAG = "CameraExtensionsViewModel"
const val REALTIME_LATENCY_UPDATE_INTERVAL_MILLIS = 1000L
}

private lateinit var cameraProvider: ProcessCameraProvider
private lateinit var extensionsManager: ExtensionsManager

Expand All @@ -69,6 +81,7 @@ class CameraExtensionsViewModel(
private var imageCapture = ImageCapture.Builder()
.setTargetAspectRatio(AspectRatio.RATIO_16_9)
.build()
private var realtimeLatencyEstimateJob: Job? = null

private val preview = Preview.Builder()
.setTargetAspectRatio(AspectRatio.RATIO_16_9)
Expand Down Expand Up @@ -127,6 +140,7 @@ class CameraExtensionsViewModel(
availableExtensions = listOf(ExtensionMode.NONE) + availableExtensions,
availableCameraLens = availableCameraLens,
extensionMode = if (availableExtensions.isEmpty()) ExtensionMode.NONE else currentCameraUiState.extensionMode,
realtimeCaptureLatencyEstimate = ImageCaptureLatencyEstimate.UNDEFINED_IMAGE_CAPTURE_LATENCY,
)
_cameraUiState.emit(newCameraUiState)
}
Expand All @@ -141,6 +155,8 @@ class CameraExtensionsViewModel(
lifecycleOwner: LifecycleOwner,
previewView: PreviewView
) {
realtimeLatencyEstimateJob?.cancel()

val currentCameraUiState = _cameraUiState.value
val cameraSelector = if (currentCameraUiState.extensionMode == ExtensionMode.NONE) {
cameraLensToSelector(currentCameraUiState.cameraLens)
Expand Down Expand Up @@ -175,30 +191,75 @@ class CameraExtensionsViewModel(
useCaseGroup
)

preview.setSurfaceProvider(previewView.surfaceProvider)
preview.surfaceProvider = previewView.surfaceProvider

viewModelScope.launch {
_cameraUiState.emit(_cameraUiState.value.copy(cameraState = CameraState.READY))
_captureUiState.emit(CaptureState.CaptureReady)
previewView.previewStreamState.asFlow().collect { previewStreamState ->
when (previewStreamState) {
PreviewView.StreamState.IDLE -> {
realtimeLatencyEstimateJob?.cancel()
realtimeLatencyEstimateJob = null
}
PreviewView.StreamState.STREAMING -> {
if (realtimeLatencyEstimateJob == null) {
realtimeLatencyEstimateJob = launch {
observeRealtimeLatencyEstimate()
}
}
}
}
}
}
}

private suspend fun CoroutineScope.observeRealtimeLatencyEstimate() {
Log.d(TAG, "Starting realtime latency estimate job")

val currentCameraUiState = _cameraUiState.value
val isSupported =
currentCameraUiState.extensionMode != ExtensionMode.NONE
&& imageCapture.realtimeCaptureLatencyEstimate != ImageCaptureLatencyEstimate.UNDEFINED_IMAGE_CAPTURE_LATENCY

if (!isSupported) {
Log.d(TAG, "Starting realtime latency estimate job: no extension mode or not supported")
_cameraUiState.emit(
_cameraUiState.value.copy(
cameraState = CameraState.PREVIEW_ACTIVE,
realtimeCaptureLatencyEstimate = ImageCaptureLatencyEstimate.UNDEFINED_IMAGE_CAPTURE_LATENCY
)
)
return
}

while (isActive) {
updateRealtimeCaptureLatencyEstimate()
delay(REALTIME_LATENCY_UPDATE_INTERVAL_MILLIS)
}
}

/**
* Stops the preview stream. This should be invoked when the captured image is displayed.
*/
fun stopPreview() {
preview.setSurfaceProvider(null)
realtimeLatencyEstimateJob?.cancel()
preview.surfaceProvider = null
viewModelScope.launch {
_cameraUiState.emit(_cameraUiState.value.copy(cameraState = CameraState.PREVIEW_STOPPED))
_cameraUiState.emit(_cameraUiState.value.copy(
cameraState = CameraState.PREVIEW_STOPPED,
realtimeCaptureLatencyEstimate = ImageCaptureLatencyEstimate.UNDEFINED_IMAGE_CAPTURE_LATENCY
))
}
}

/**
* Toggle the camera lens face. This has no effect if there is only one available camera lens.
*/
fun switchCamera() {
realtimeLatencyEstimateJob?.cancel()
val currentCameraUiState = _cameraUiState.value
if (currentCameraUiState.cameraState == CameraState.READY) {
if (currentCameraUiState.cameraState == CameraState.READY || currentCameraUiState.cameraState == CameraState.PREVIEW_ACTIVE) {
// To switch the camera lens, there has to be at least 2 camera lenses
if (currentCameraUiState.availableCameraLens.size == 1) return

Expand Down Expand Up @@ -230,6 +291,7 @@ class CameraExtensionsViewModel(
* exception containing more details on the reason for failure.
*/
fun capturePhoto() {
realtimeLatencyEstimateJob?.cancel()
viewModelScope.launch {
_captureUiState.emit(CaptureState.CaptureStarted)
}
Expand Down Expand Up @@ -306,6 +368,7 @@ class CameraExtensionsViewModel(
_cameraUiState.value.copy(
cameraState = CameraState.NOT_READY,
extensionMode = extensionMode,
realtimeCaptureLatencyEstimate = ImageCaptureLatencyEstimate.UNDEFINED_IMAGE_CAPTURE_LATENCY,
)
)
_captureUiState.emit(CaptureState.CaptureNotReady)
Expand All @@ -331,4 +394,18 @@ class CameraExtensionsViewModel(
CameraSelector.LENS_FACING_BACK -> CameraSelector.DEFAULT_BACK_CAMERA
else -> throw IllegalArgumentException("Invalid lens facing type: $lensFacing")
}

private suspend fun updateRealtimeCaptureLatencyEstimate() {
val estimate = imageCapture.realtimeCaptureLatencyEstimate
Log.d(TAG, "Realtime capture latency estimate: $estimate")
if (estimate == ImageCaptureLatencyEstimate.UNDEFINED_IMAGE_CAPTURE_LATENCY) {
return
}
_cameraUiState.emit(
_cameraUiState.value.copy(
cameraState = CameraState.PREVIEW_ACTIVE,
realtimeCaptureLatencyEstimate = estimate
)
)
}
}
Loading
Loading