CameraX qrcode scanner detect wrong

Question

Hi i develop a simple QRcode scanner with CameraX. It works but i'd like show a preiew shape around qrcode. I create a custom view and send boundBox of barcode but.. dimensions and position are wrong.

I think that it's a coordinate translate problems.. maybe :(

here a little project https://github.com/giuseppesorce/cameraxscan

Some code:

package com.gs.scancamerax

import android.Manifest.permission.CAMERA
import android.annotation.SuppressLint
import android.content.pm.PackageManager
import android.os.Bundle
import android.util.DisplayMetrics
import android.util.Log
import android.view.LayoutInflater
import android.view.View
import android.view.ViewGroup
import android.widget.Toast
import androidx.camera.core.*
import androidx.camera.lifecycle.ProcessCameraProvider
import androidx.core.content.ContextCompat
import androidx.fragment.app.Fragment
import com.google.mlkit.vision.barcode.BarcodeScanning
import com.google.mlkit.vision.common.InputImage
import com.gs.scancamerax.databinding.FragmentScanBarcodeBinding


import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicBoolean
import kotlin.math.abs
import kotlin.math.max
import kotlin.math.min

typealias BarcodeListener = (barcode: String) -> Unit


class ScanBarcodeFragment : Fragment() {
    private var scanningResultListener: ScanningResultListener? = null
    private var flashEnabled: Boolean = false
    private var camera: Camera? = null
    private var processingBarcode = AtomicBoolean(false)
    private lateinit var cameraExecutor: ExecutorService
    private var _binding: FragmentScanBarcodeBinding? = null
    private val binding get() = _binding!!
    private val TAG = "CameraXBasic"
    private val RATIO_4_3_VALUE = 4.0 / 3.0
    private val RATIO_16_9_VALUE = 16.0 / 9.0
    private var imageCapture: ImageCapture? = null
    private var imageAnalyzer: ImageAnalysis? = null
    private var cameraProvider: ProcessCameraProvider? = null


    override fun onCreateView(
        inflater: LayoutInflater,
        container: ViewGroup?,
        savedInstanceState: Bundle?
    ): View? {
        _binding = FragmentScanBarcodeBinding.inflate(inflater, container, false)
        val view = binding.root
        return view
    }

    override fun onResume() {
        super.onResume()
        processingBarcode.set(false)
       initFragment()
    }


    private fun startCamera() {
        val cameraProviderFuture = ProcessCameraProvider.getInstance(requireContext())
        cameraProviderFuture.addListener(Runnable {
            // CameraProvider
            cameraProvider = cameraProviderFuture.get()
            // Build and bind the camera use cases
            bindCameraUseCases()
        }, ContextCompat.getMainExecutor(requireContext()))
    }



    private fun aspectRatio(width: Int, height: Int): Int {
        val previewRatio = max(width, height).toDouble() / min(width, height)
        if (abs(previewRatio - RATIO_4_3_VALUE) <= abs(previewRatio - RATIO_16_9_VALUE)) {
            return AspectRatio.RATIO_4_3
        }
        return AspectRatio.RATIO_16_9
    }

    private var preview: Preview? = null
    private fun bindCameraUseCases() {

        // Get screen metrics used to setup camera for full screen resolution
        val metrics =
            DisplayMetrics().also { binding.fragmentScanBarcodePreviewView.display.getRealMetrics(it) }
        Log.d(TAG, "Screen metrics: ${metrics.widthPixels} x ${metrics.heightPixels}")

        val screenAspectRatio = aspectRatio(metrics.widthPixels, metrics.heightPixels)
        Log.d(TAG, "Preview aspect ratio: $screenAspectRatio")

        val rotation = binding.fragmentScanBarcodePreviewView.display.rotation

        // CameraProvider
        val cameraProvider = cameraProvider
            ?: throw IllegalStateException("Camera initialization failed.")

        // CameraSelector
        val cameraSelector = CameraSelector.Builder().requireLensFacing(lensFacing).build()

        // Preview
        preview = Preview.Builder()
            // We request aspect ratio but no resolution
            .setTargetAspectRatio(screenAspectRatio)
            // Set initial target rotation
            .setTargetRotation(rotation)
            .build()

        // ImageCapture
        imageCapture = ImageCapture.Builder()
            .setCaptureMode(ImageCapture.CAPTURE_MODE_MINIMIZE_LATENCY)
            // We request aspect ratio but no resolution to match preview config, but letting
            // CameraX optimize for whatever specific resolution best fits our use cases
            .setTargetAspectRatio(screenAspectRatio)
            // Set initial target rotation, we will have to call this again if rotation changes
            // during the lifecycle of this use case
            .setTargetRotation(rotation)
            .build()

        // ImageAnalysis
        imageAnalyzer = ImageAnalysis.Builder()
            // We request aspect ratio but no resolution
            .setTargetAspectRatio(screenAspectRatio)
            // Set initial target rotation, we will have to call this again if rotation changes
            // during the lifecycle of this use case
            .setTargetRotation(rotation)
            .build()
            // The analyzer can then be assigned to the instance
            .also {
                it.setAnalyzer(cameraExecutor, BarcodeAnalyzer { luma ->
                    // Values returned from our analyzer are passed to the attached listener
                    // We log image analysis results here - you should do something useful
                    // instead!
                    Log.d(TAG, "Average luminosity: $luma")
                })
            }

        // Must unbind the use-cases before rebinding them
        cameraProvider.unbindAll()

        try {
            // A variable number of use-cases can be passed here -
            // camera provides access to CameraControl & CameraInfo
            camera = cameraProvider.bindToLifecycle(
                this, cameraSelector, preview, imageCapture, imageAnalyzer
            )

            // Attach the viewfinder's surface provider to preview use case
            preview?.setSurfaceProvider(binding.fragmentScanBarcodePreviewView.surfaceProvider)
        } catch (exc: Exception) {
            Log.e(TAG, "Use case binding failed", exc)
        }
    }


    private var lensFacing: Int = CameraSelector.LENS_FACING_BACK


    private fun allPermissionsGranted() = REQUIRED_PERMISSIONS.all {
        ContextCompat.checkSelfPermission(
            requireContext(), it
        ) == PackageManager.PERMISSION_GRANTED
    }

     fun initFragment() {
        cameraExecutor = Executors.newSingleThreadExecutor()
        if (allPermissionsGranted()) {
            binding.fragmentScanBarcodePreviewView.post {
                startCamera()
            }
        } else {
            requestPermissions(REQUIRED_PERMISSIONS, REQUEST_CODE_PERMISSIONS)
        }
    }

    override fun onRequestPermissionsResult(
        requestCode: Int, permissions: Array<String>,
        grantResults: IntArray
    ) {
        if (requestCode == REQUEST_CODE_PERMISSIONS) {
            if (allPermissionsGranted()) {
                startCamera()
            } else {
                activity?.let {
                    Toast.makeText(
                        it.applicationContext,
                        "Permissions not granted by the user.",
                        Toast.LENGTH_SHORT
                    ).show()
                }
            }
        }
    }

    private fun searchBarcode(barcode: String) {
        Log.e("driver", "searchBarcode: $barcode")
    }

    override fun onDestroy() {
        cameraExecutor.shutdown()
        camera = null
        _binding = null
        super.onDestroy()
    }


    inner class BarcodeAnalyzer(private val barcodeListener: BarcodeListener) :
        ImageAnalysis.Analyzer {

        private val scanner = BarcodeScanning.getClient()
        @SuppressLint("UnsafeExperimentalUsageError")
        override fun analyze(imageProxy: ImageProxy) {
            val mediaImage = imageProxy.image
            if (mediaImage != null) {
                val image =
                    InputImage.fromMediaImage(mediaImage, imageProxy.imageInfo.rotationDegrees)
                // Pass image to the scanner and have it do its thing
                scanner.process(image)
                    .addOnSuccessListener { barcodes ->

                        for (barcode in barcodes) {

                            barcodeListener(barcode.rawValue ?: "")
                            binding.myView.setBounds(barcode.boundingBox)
                        }
                    }
                    .addOnFailureListener {
                        // You should really do something about Exceptions
                    }
                    .addOnCompleteListener {
                        // It's important to close the imageProxy
                        imageProxy.close()
                    }
            }
        }
    }

    override fun onDestroyView() {
        super.onDestroyView()
        _binding = null
    }

    fun setScanResultListener(listener: ScanningResultListener) {
        this.scanningResultListener = listener
    }

    companion object {
        private val REQUIRED_PERMISSIONS = arrayOf(CAMERA)
        private const val REQUEST_CODE_PERMISSIONS = 10
        const val TAG = "BarCodeFragment"

        @JvmStatic
        fun newInstance() = ScanBarcodeFragment()
    }
}

Answer 1

To have coordinate translation between the image captured from the camera and the preview view on screen, it's easiest to use instances of LifecycleCameraController and PreviewView from CameraX. Then pass an instance of MLKitAnalyzer using LifecycleCameraController.setImageAnalysisAnalyzer(). You can then pass CameraController.COORDINATE_SYSTEM_VIEW_REFERENCED as targetCoordinateSystem to the MlKitAnalyzer and the Barcode.boundingBox will be automatically translated to screen coordinates in PreviewView. Note that the bounding box coordinates will still be in pixels, so you need to translate them further to dp (e.g. in Compose using LocalDensity.current and the toDp() extension function).

Example solution (in Compose):

@Composable
fun CameraScanner(
    onBarcodeScanned: (Barcode) -> Unit,
    modifier: Modifier = Modifier,
) {
    val localContext = LocalContext.current
    val lifecycleOwner = LocalLifecycleOwner.current
    val localDensity = LocalDensity.current

    val lifecycleCameraController = remember {
        LifecycleCameraController(localContext).apply {
            bindToLifecycle(lifecycleOwner)
            cameraSelector = CameraSelector.DEFAULT_BACK_CAMERA
        }
    }

    val previewView = remember {
        PreviewView(localContext).apply {
            controller = lifecycleCameraController
        }
    }

    var barcodeFrameX: Dp by remember { mutableStateOf(0.dp) }
    var barcodeFrameY: Dp by remember { mutableStateOf(0.dp) }
    var barcodeFrameWidth: Dp by remember { mutableStateOf(0.dp) }
    var barcodeFrameHeight: Dp by remember { mutableStateOf(0.dp) }

    LaunchedEffect(previewView) {
        val barcodeScanner = BarcodeScanning.getClient(
            BarcodeScannerOptions.Builder()
                .setBarcodeFormats(Barcode.FORMAT_ALL_FORMATS)
                .build()
        )

        val mlKitAnalyzer = MlKitAnalyzer(
            listOf(barcodeScanner),
            CameraController.COORDINATE_SYSTEM_VIEW_REFERENCED,
            ContextCompat.getMainExecutor(localContext),
        ) { result ->
            result.getValue(barcodeScanner)?.filter { !it.rawValue.isNullOrEmpty() }
                ?.forEach { barcode ->
                    onBarcodeScanned(barcode)

                    barcode.boundingBox?.run {
                        barcodeFrameX = with(localDensity) { left.toDp() }
                        barcodeFrameY = with(localDensity) { top.toDp() }
                        barcodeFrameWidth = with(localDensity) { width().toDp() }
                        barcodeFrameHeight = with(localDensity) { height().toDp() }
                    }
                }
        }

        lifecycleCameraController.apply {
            setImageAnalysisAnalyzer(
                Executors.newSingleThreadExecutor(),
                mlKitAnalyzer,
            )
            setEnabledUseCases(CameraController.IMAGE_ANALYSIS)
        }
    }

    Box {
        AndroidView(
            modifier = modifier.fillMaxSize(),
            factory = { previewView },
        )
        Box(
            modifier = Modifier
                .offset(x = barcodeFrameX, y = barcodeFrameY)
                .width(barcodeFrameWidth)
                .height(barcodeFrameHeight)
                .border(width = 2.dp, color = Color.Red)
        )
    }
}

Also this article was very helpful for understanding the separation of concerns between CameraController and PreviewView CameraX APIs. Especially the fact that they were designed with WYSIWYG (What You See Is What You Get) principle in mind to make developers' lives easier.

Answer 2

Not sure if you are still having the problem, but now CameraX provides a helper class called CoordinateTransform that transforms the coordinates from one use case to another. For example, you can transform the bounding box of the QR code detected from ImageAnalysis to PreviewView coordinates, and draw a bounding box there.

// ImageProxy is the output of an ImageAnalysis.
OutputTransform source = ImageProxyTransformFactory().getOutputTransform(imageProxy);
OutputTransform target = previewView.getOutputTransform();

// Build the transform from ImageAnalysis to PreviewView
CoordinateTransform coordinateTransform = new CoordinateTransform(source, target);

// Detect barcode in ImageProxy and transform the coordinates to PreviewView.
RectF boundingBox = detectBarcodeInImageProxy(imageProxy);
coordinateTransform.mapRect(boundingBox);

// Now boundingBox contains the coordinates of the bounding box in PreviewView.