I am trying to integrate MediaPipe Poselandmarker in my flutter app using native communication with the help of method channel. The problem is when the camera is getting opened,it is being displayed over top of overlay. But i want overlay to be displayed over top of camera. I am sharing my kotlin code for reference. Please tell me where I am doing wrong.
This is my NativeView.kt:, In this I am setting up fragment container
private fun setupFragmentContainer(context: Context) {
Log.d("NativeView", "Setting up FragmentContainerView for CameraFragment")
// Create a FragmentContainerView dynamically if not already added.
val fragmentContainerView = FragmentContainerView(context).apply {
id = View.generateViewId() // Generate a unique ID for the container
layoutParams = LayoutParams(
LayoutParams.MATCH_PARENT,
LayoutParams.MATCH_PARENT
)
}
this.addView(fragmentContainerView)
// Use FragmentManager to add or replace the fragment
val fragmentActivity = activity as? FragmentActivity ?: throw ClassCastException("Activity must be a FragmentActivity")
val fragmentTransaction = fragmentActivity.supportFragmentManager.beginTransaction()
// Check if CameraFragment is already added
val existingFragment = fragmentActivity.supportFragmentManager.findFragmentByTag("CameraFragment")
if (existingFragment == null) {
fragmentTransaction.replace(fragmentContainerView.id, CameraFragment(), "CameraFragment")
fragmentTransactionmit()
Log.d("NativeView", "CameraFragment added to FragmentContainerView")
} else {
Log.d("NativeView", "CameraFragment already exists, no need to add again")
}
// After fragment is added, bring overlays to the front
// val overlayView = findViewById<OverlayView>(R.id.overlay) // Ensure this ID is correct
// overlayView.bringToFront()
//
// val bottomSheetLayout = findViewById<View>(R.id.bottom_sheet_layout)
// bottomSheetLayout.bringToFront()
//
// // Optionally, invalidate the views to make sure they get redrawn
// overlayView.invalidate()
// bottomSheetLayout.invalidate()
}
This is my CameraFragmemt.kt
/*
* Copyright 2023 The TensorFlow Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* .0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package app.peak.med.bibo.fragment
import android.annotation.SuppressLint
import android.content.res.Configuration
import android.os.Bundle
import android.util.Log
import android.view.LayoutInflater
import android.view.View
import android.view.ViewGroup
import android.widget.AdapterView
import android.widget.Toast
import androidx.camera.core.Preview
import androidx.camera.core.CameraSelector
import androidx.camera.core.ImageAnalysis
import androidx.camera.core.ImageProxy
import androidx.camera.core.Camera
import androidx.camera.core.AspectRatio
import androidx.camera.lifecycle.ProcessCameraProvider
import androidx.core.content.ContextCompat
import androidx.fragment.app.Fragment
import androidx.fragment.app.activityViewModels
import androidx.navigation.Navigation
import app.peak.med.bibo.PoseLandmarkerHelper
import app.peak.med.bibo.MainViewModel
import app.peak.med.bibo.OverlayView
import app.peak.med.bibo.R
import app.peak.med.bibo.databinding.FragmentCameraBinding
import com.google.mediapipe.tasks.vision.core.RunningMode
import java.util.Locale
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
class CameraFragment : Fragment(), PoseLandmarkerHelper.LandmarkerListener {
companion object {
private const val TAG = "CAMERA FRAGMENT"
}
private var _fragmentCameraBinding: FragmentCameraBinding? = null
private val fragmentCameraBinding
get() = _fragmentCameraBinding!!
private lateinit var poseLandmarkerHelper: PoseLandmarkerHelper
private val viewModel: MainViewModel by activityViewModels()
private var preview: Preview? = null
private var imageAnalyzer: ImageAnalysis? = null
private var camera: Camera? = null
private var cameraProvider: ProcessCameraProvider? = null
private var cameraFacing = CameraSelector.LENS_FACING_FRONT
/** Blocking ML operations are performed using this executor */
private lateinit var backgroundExecutor: ExecutorService
override fun onResume() {
super.onResume()
// Make sure that all permissions are still present, since the
// user could have removed them while the app was in paused state.
if (!PermissionsFragment.hasPermissions(requireContext())) {
Navigation.findNavController(
requireActivity(), R.id.fragment_container
).navigate(R.id.action_camera_to_permissions)
}
// Start the PoseLandmarkerHelper again when users come back
// to the foreground.
backgroundExecutor.execute {
if(this::poseLandmarkerHelper.isInitialized) {
if (poseLandmarkerHelper.isClose()) {
poseLandmarkerHelper.setupPoseLandmarker()
}
}
}
}
override fun onPause() {
super.onPause()
if(this::poseLandmarkerHelper.isInitialized) {
viewModel.setMinPoseDetectionConfidence(poseLandmarkerHelper.minPoseDetectionConfidence)
viewModel.setMinPoseTrackingConfidence(poseLandmarkerHelper.minPoseTrackingConfidence)
viewModel.setMinPosePresenceConfidence(poseLandmarkerHelper.minPosePresenceConfidence)
viewModel.setDelegate(poseLandmarkerHelper.currentDelegate)
// Close the PoseLandmarkerHelper and release resources
backgroundExecutor.execute { poseLandmarkerHelper.clearPoseLandmarker() }
}
}
override fun onDestroyView() {
_fragmentCameraBinding = null
super.onDestroyView()
// Shut down our background executor
backgroundExecutor.shutdown()
backgroundExecutor.awaitTermination(
Long.MAX_VALUE, TimeUnit.NANOSECONDS
)
}
override fun onCreateView(
inflater: LayoutInflater,
container: ViewGroup?,
savedInstanceState: Bundle?
): View {
_fragmentCameraBinding =
FragmentCameraBinding.inflate(inflater, container, false)
return fragmentCameraBinding.root
}
@SuppressLint("MissingPermission")
override fun onViewCreated(view: View, savedInstanceState: Bundle?) {
super.onViewCreated(view, savedInstanceState)
val overlayView = view.findViewById<OverlayView>(R.id.overlay)
overlayView.bringToFront()
// If needed, ensure bottom sheet layout is on top of both camera and overlay
// val bottomSheetLayout = view.findViewById<View>(R.id.bottom_sheet_layout)
// bottomSheetLayout.bringToFront()
view.requestLayout()
// Initialize our background executor
backgroundExecutor = Executors.newSingleThreadExecutor()
// Wait for the views to be properly laid out
fragmentCameraBinding.viewFinder.post {
// Set up the camera and its use cases
setUpCamera()
overlayView.bringToFront()
//fragmentCameraBinding.overlay.bringToFront() // Ensure overlay is above camera view.
}
// Create the PoseLandmarkerHelper that will handle the inference
backgroundExecutor.execute {
poseLandmarkerHelper = PoseLandmarkerHelper(
context = requireContext(),
runningMode = RunningMode.LIVE_STREAM,
minPoseDetectionConfidence = viewModel.currentMinPoseDetectionConfidence,
minPoseTrackingConfidence = viewModel.currentMinPoseTrackingConfidence,
minPosePresenceConfidence = viewModel.currentMinPosePresenceConfidence,
currentDelegate = viewModel.currentDelegate,
poseLandmarkerHelperListener = this
)
}
// Attach listeners to UI control widgets
// initBottomSheetControls()
}
/*private fun initBottomSheetControls() {
// init bottom sheet settings
fragmentCameraBinding.bottomSheetLayout.detectionThresholdValue.text =
String.format(
Locale.US, "%.2f", viewModel.currentMinPoseDetectionConfidence
)
fragmentCameraBinding.bottomSheetLayout.trackingThresholdValue.text =
String.format(
Locale.US, "%.2f", viewModel.currentMinPoseTrackingConfidence
)
fragmentCameraBinding.bottomSheetLayout.presenceThresholdValue.text =
String.format(
Locale.US, "%.2f", viewModel.currentMinPosePresenceConfidence
)
// When clicked, lower pose detection score threshold floor
fragmentCameraBinding.bottomSheetLayout.detectionThresholdMinus.setOnClickListener {
if (poseLandmarkerHelper.minPoseDetectionConfidence >= 0.2) {
poseLandmarkerHelper.minPoseDetectionConfidence -= 0.1f
updateControlsUi()
}
}
// When clicked, raise pose detection score threshold floor
fragmentCameraBinding.bottomSheetLayout.detectionThresholdPlus.setOnClickListener {
if (poseLandmarkerHelper.minPoseDetectionConfidence <= 0.8) {
poseLandmarkerHelper.minPoseDetectionConfidence += 0.1f
updateControlsUi()
}
}
// When clicked, lower pose tracking score threshold floor
fragmentCameraBinding.bottomSheetLayout.trackingThresholdMinus.setOnClickListener {
if (poseLandmarkerHelper.minPoseTrackingConfidence >= 0.2) {
poseLandmarkerHelper.minPoseTrackingConfidence -= 0.1f
updateControlsUi()
}
}
// When clicked, raise pose tracking score threshold floor
fragmentCameraBinding.bottomSheetLayout.trackingThresholdPlus.setOnClickListener {
if (poseLandmarkerHelper.minPoseTrackingConfidence <= 0.8) {
poseLandmarkerHelper.minPoseTrackingConfidence += 0.1f
updateControlsUi()
}
}
// When clicked, lower pose presence score threshold floor
fragmentCameraBinding.bottomSheetLayout.presenceThresholdMinus.setOnClickListener {
if (poseLandmarkerHelper.minPosePresenceConfidence >= 0.2) {
poseLandmarkerHelper.minPosePresenceConfidence -= 0.1f
updateControlsUi()
}
}
// When clicked, raise pose presence score threshold floor
fragmentCameraBinding.bottomSheetLayout.presenceThresholdPlus.setOnClickListener {
if (poseLandmarkerHelper.minPosePresenceConfidence <= 0.8) {
poseLandmarkerHelper.minPosePresenceConfidence += 0.1f
updateControlsUi()
}
}
// When clicked, change the underlying hardware used for inference.
// Current options are CPU and GPU
fragmentCameraBinding.bottomSheetLayout.spinnerDelegate.setSelection(
viewModel.currentDelegate, false
)
fragmentCameraBinding.bottomSheetLayout.spinnerDelegate.onItemSelectedListener =
object : AdapterView.OnItemSelectedListener {
override fun onItemSelected(
p0: AdapterView<*>?, p1: View?, p2: Int, p3: Long
) {
try {
poseLandmarkerHelper.currentDelegate = p2
updateControlsUi()
} catch(e: UninitializedPropertyAccessException) {
Log.e(TAG, "PoseLandmarkerHelper has not been initialized yet.")
}
}
override fun onNothingSelected(p0: AdapterView<*>?) {
/* no op */
}
}
// When clicked, change the underlying model used for object detection
fragmentCameraBinding.bottomSheetLayout.spinnerModel.setSelection(
viewModel.currentModel,
false
)
fragmentCameraBinding.bottomSheetLayout.spinnerModel.onItemSelectedListener =
object : AdapterView.OnItemSelectedListener {
override fun onItemSelected(
p0: AdapterView<*>?,
p1: View?,
p2: Int,
p3: Long
) {
poseLandmarkerHelper.currentModel = p2
updateControlsUi()
}
override fun onNothingSelected(p0: AdapterView<*>?) {
/* no op */
}
}
}*/
// Update the values displayed in the bottom sheet. Reset Poselandmarker
// helper.
/*private fun updateControlsUi() {
if(this::poseLandmarkerHelper.isInitialized) {
fragmentCameraBinding.bottomSheetLayout.detectionThresholdValue.text =
String.format(
Locale.US,
"%.2f",
poseLandmarkerHelper.minPoseDetectionConfidence
)
fragmentCameraBinding.bottomSheetLayout.trackingThresholdValue.text =
String.format(
Locale.US,
"%.2f",
poseLandmarkerHelper.minPoseTrackingConfidence
)
fragmentCameraBinding.bottomSheetLayout.presenceThresholdValue.text =
String.format(
Locale.US,
"%.2f",
poseLandmarkerHelper.minPosePresenceConfidence
)
// Needs to be cleared instead of reinitialized because the GPU
// delegate needs to be initialized on the thread using it when applicable
backgroundExecutor.execute {
poseLandmarkerHelper.clearPoseLandmarker()
poseLandmarkerHelper.setupPoseLandmarker()
}
fragmentCameraBinding.overlay.clear()
}
}*/
// Initialize CameraX, and prepare to bind the camera use cases
private fun setUpCamera() {
val cameraProviderFuture =
ProcessCameraProvider.getInstance(requireContext())
cameraProviderFuture.addListener(
{
// CameraProvider
cameraProvider = cameraProviderFuture.get()
// Build and bind the camera use cases
bindCameraUseCases()
}, ContextCompat.getMainExecutor(requireContext())
)
}
// Declare and bind preview, capture and analysis use cases
@SuppressLint("UnsafeOptInUsageError")
private fun bindCameraUseCases() {
// CameraProvider
val cameraProvider = cameraProvider
?: throw IllegalStateException("Camera initialization failed.")
val cameraSelector =
CameraSelector.Builder().requireLensFacing(cameraFacing).build()
// Preview. Only using the 4:3 ratio because this is the closest to our models
preview = Preview.Builder().setTargetAspectRatio(AspectRatio.RATIO_4_3)
.setTargetRotation(fragmentCameraBinding.viewFinder.display.rotation)
.build()
// ImageAnalysis. Using RGBA 8888 to match how our models work
imageAnalyzer =
ImageAnalysis.Builder().setTargetAspectRatio(AspectRatio.RATIO_4_3)
.setTargetRotation(fragmentCameraBinding.viewFinder.display.rotation)
.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
.setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_RGBA_8888)
.build()
// The analyzer can then be assigned to the instance
.also {
it.setAnalyzer(backgroundExecutor) { image ->
detectPose(image)
}
}
// Must unbind the use-cases before rebinding them
cameraProvider.unbindAll()
try {
// A variable number of use-cases can be passed here -
// camera provides access to CameraControl & CameraInfo
camera = cameraProvider.bindToLifecycle(
this, cameraSelector, preview, imageAnalyzer
)
// Attach the viewfinder's surface provider to preview use case
preview?.setSurfaceProvider(fragmentCameraBinding.viewFinder.surfaceProvider)
// val overlayView: View = view.findViewById(R.id.overlay) // This can be any view like lines or annotations
// overlayView.bringToFront()
val overlayView: View? = view?.findViewById(R.id.overlay)!!
overlayView?.let {
// Do something with the overlay view, e.g., bring it to the front
it.bringToFront()
}
} catch (exc: Exception) {
Log.e(TAG, "Use case binding failed", exc)
}
}
private fun detectPose(imageProxy: ImageProxy) {
if(this::poseLandmarkerHelper.isInitialized) {
poseLandmarkerHelper.detectLiveStream(
imageProxy = imageProxy,
isFrontCamera = cameraFacing == CameraSelector.LENS_FACING_FRONT
)
}
}
override fun onConfigurationChanged(newConfig: Configuration) {
super.onConfigurationChanged(newConfig)
imageAnalyzer?.targetRotation =
fragmentCameraBinding.viewFinder.display.rotation
}
// Update UI after pose have been detected. Extracts original
// image height/width to scale and place the landmarks properly through
// OverlayView
override fun onResults(
resultBundle: PoseLandmarkerHelper.ResultBundle
) {
activity?.runOnUiThread {
if (_fragmentCameraBinding != null) {
// fragmentCameraBinding.bottomSheetLayout.inferenceTimeVal.text =
// String.format("%d ms", resultBundle.inferenceTime)
//fragmentCameraBinding.overlay.bringToFront()
// Pass necessary information to OverlayView for drawing on the canvas
fragmentCameraBinding.overlay.setResults(
resultBundle.results.first(),
resultBundle.inputImageHeight,
resultBundle.inputImageWidth,
RunningMode.LIVE_STREAM
)
// Force a redraw
fragmentCameraBinding.overlay.invalidate()
}
}
}
override fun onError(error: String, errorCode: Int) {
activity?.runOnUiThread {
Toast.makeText(requireContext(), error, Toast.LENGTH_SHORT).show()
if (errorCode == PoseLandmarkerHelper.GPU_ERROR) {
// fragmentCameraBinding.bottomSheetLayout.spinnerDelegate.setSelection(
// PoseLandmarkerHelper.DELEGATE_CPU, false
//)
}
}
}
}
In MainActivity.kt, when I am running the below code, overlay is getting built on top of camera
class MainActivity : AppCompatActivity() {
private lateinit var activityMainBinding: ActivityMainBinding
private val viewModel : MainViewModel by viewModels()
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
activityMainBinding = ActivityMainBinding.inflate(layoutInflater)
setContentView(activityMainBinding.root)
val navHostFragment =
supportFragmentManager.findFragmentById(R.id.fragment_container) as NavHostFragment
val navController = navHostFragment.navController
activityMainBinding.navigation.setupWithNavController(navController)
activityMainBinding.navigation.setOnNavigationItemReselectedListener {
// ignore the reselection
}
}
override fun onBackPressed() {
finish()
}
}
But when I have changed it to flutterFragmentActivity, above problem is coming
class MainActivity : FlutterFragmentActivity() { //class MainActivity : FlutterActivity() {
companion object {
private const val TAG = "MainActivity"
}
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
// No need to access flutterEngine here; it will be configured below.
}
override fun configureFlutterEngine(flutterEngine: FlutterEngine) {
super.configureFlutterEngine(flutterEngine)
try {
flutterEngine.plugins.add(FlutterMediapipePlugin())
} catch (e: Exception) {
Log.e(TAG, "Error registering plugin flutter_mediapipe", e)
}
}
}