CODEWITHSUNDEEP
androidfirebase-mlkitandroid-camerax
vrgrg
vrgrg

Reputation: 616

CameraX analyzer does not run on every frame

I tried to implement MLKit Text analyzer using CameraX using Android developer documentation. My expectation was that the analyzer will run on each frame, but it runs only once, when the Preview is initialized. The text is analyzed by the MLKit, but the analysis is executed only if I rotate the screen and restart the fragment. Why this happens?

In the XML I use <androidx.camera.view.PreviewView .../>.

In Fragment:

private lateinit var cameraProviderFuture: ListenableFuture<ProcessCameraProvider>

private fun startCamera() {
    context?.let { ctx ->
        cameraProviderFuture = ProcessCameraProvider.getInstance(ctx)
        cameraProviderFuture.addListener(Runnable {
            val cameraProvider = cameraProviderFuture.get()
            bindPreview(cameraProvider)
        }, ContextCompat.getMainExecutor(ctx))
    }
}

private fun bindPreview(cameraProvider: ProcessCameraProvider) {
    val preview: Preview = Preview.Builder()
        .build()

    val cameraSelector: CameraSelector = CameraSelector.Builder()
        .requireLensFacing(CameraSelector.LENS_FACING_BACK)
        .build()

    val analyzer = ImageAnalysis.Builder()
        .setTargetResolution(Size(1280, 720))
        .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
        .build().apply {
            setAnalyzer(Executors.newSingleThreadExecutor(), TextAnalyzer())
        }

    val camera = cameraProvider.bindToLifecycle(viewLifecycleOwner, cameraSelector, analyzer, preview)

    preview.setSurfaceProvider(preview_view.createSurfaceProvider(camera.cameraInfo))
}

Analyzer implementation:

class TextAnalyzer : ImageAnalysis.Analyzer {

val detector = FirebaseVision.getInstance().onDeviceTextRecognizer

@SuppressLint("UnsafeExperimentalUsageError")
override fun analyze(imageProxy: ImageProxy) {
    val mediaImage = imageProxy.image
    val rotation = degreesToFirebaseRotation(imageProxy.imageInfo.rotationDegrees)
    mediaImage?.let { image ->
        val firebaseImage = FirebaseVisionImage.fromMediaImage(image, rotation)
        detector.processImage(firebaseImage)
            .addOnSuccessListener { firebaseVisionText ->
                val text = firebaseVisionText.text
                val textBlock = firebaseVisionText.textBlocks
                Log.i("TEXT_ANALYZER", "success text $text")
                textBlock.forEach {
                    Log.i("TEXT_ANALYZER", "success text BLOCK $it")
                }
            }
            .addOnFailureListener { e ->
                Log.i("TEXT_ANALYZER", "failed ex $e")
            }
    }
}

private fun degreesToFirebaseRotation(degrees: Int): Int = when (degrees) {
    0 -> ROTATION_0
    90 -> ROTATION_90
    180 -> ROTATION_180
    270 -> ROTATION_270
    else -> throw Exception("Rotation must be 0, 90, 180, or 270.")
}
}

Upvotes: 3

Views: 2227

Answers (1)

vrgrg
vrgrg

Reputation: 616

I have figured out the answer. I just had to read the docs more carefully, it says that the image proxy has to be closed in order not to block the thread and the Preview :-) 

  override fun analyze(imageProxy: ImageProxy) {
        val mediaImage = imageProxy.image
        val rotation = degreesToFirebaseRotation(imageProxy.imageInfo.rotationDegrees)
        mediaImage?.let { image ->
            val firebaseImage = FirebaseVisionImage.fromMediaImage(image, rotation)
            detector.processImage(firebaseImage)
                .addOnSuccessListener { firebaseVisionText ->
                    val text = firebaseVisionText.text
                    val textBlock = firebaseVisionText.textBlocks
                    // IMPORTANT
                    imageProxy.close()
                }
                .addOnFailureListener { e ->
                    // IMPORTANT
                    imageProxy.close()
                }
        }
    }

Upvotes: 16

Related Questions