From 94938a295bb2ade0d08da6397cc969cd7513cf1b Mon Sep 17 00:00:00 2001 From: Vishal Rabadiya Date: Thu, 8 May 2025 14:26:48 +0530 Subject: [PATCH 1/2] Added Participant View, Added Daily custom track support, Added extra parameters like presentationTimeStamp, frameNumber and startTimestamp. --- android/build.gradle | 3 +- .../com/tsmediapipe/PoseLandmarkerHelper.kt | 58 +++++++++++++++- .../tsmediapipe/fragment/CameraFragment.kt | 14 ++-- .../ParticipantOverlayManager.kt | 69 +++++++++++++++++++ .../main/res/layout/fragment_my_camera.xml | 7 ++ .../src/main/res/layout/participant_view.xml | 12 ++++ 6 files changed, 157 insertions(+), 6 deletions(-) create mode 100644 android/src/main/java/com/tsmediapipe/participantView/ParticipantOverlayManager.kt create mode 100644 android/src/main/res/layout/participant_view.xml diff --git a/android/build.gradle b/android/build.gradle index 359f843..9ec283f 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -96,7 +96,8 @@ def kotlin_version = getExtOrDefault("kotlinVersion") dependencies { // For < 0.71, this will be from the local maven repo // For > 0.71, this will be replaced by `com.facebook.react:react-android:$version` by react gradle plugin - + implementation 'co.daily:client:0.29.0' + implementation 'com.google.code.gson:gson:2.11.0' //noinspection GradleDynamicVersion diff --git a/android/src/main/java/com/tsmediapipe/PoseLandmarkerHelper.kt b/android/src/main/java/com/tsmediapipe/PoseLandmarkerHelper.kt index 1dc36b8..0e507fd 100644 --- a/android/src/main/java/com/tsmediapipe/PoseLandmarkerHelper.kt +++ b/android/src/main/java/com/tsmediapipe/PoseLandmarkerHelper.kt @@ -1,12 +1,16 @@ package com.tsmediapipe import android.content.Context +import android.content.Intent import android.graphics.Bitmap import android.graphics.Matrix +import android.os.Handler +import android.os.HandlerThread import android.os.SystemClock import android.util.Log import androidx.annotation.VisibleForTesting import androidx.camera.core.ImageProxy +import androidx.localbroadcastmanager.content.LocalBroadcastManager import com.google.mediapipe.framework.image.BitmapImageBuilder import com.google.mediapipe.framework.image.MPImage import com.google.mediapipe.tasks.core.BaseOptions @@ -14,6 +18,9 @@ import com.google.mediapipe.tasks.core.Delegate import com.google.mediapipe.tasks.vision.core.RunningMode import com.google.mediapipe.tasks.vision.poselandmarker.PoseLandmarker import com.google.mediapipe.tasks.vision.poselandmarker.PoseLandmarkerResult +import java.io.ByteArrayOutputStream +import co.daily.model.customtrack.CustomVideoSourceSurface +import java.lang.reflect.Field class PoseLandmarkerHelper( var minPoseDetectionConfidence: Float = DEFAULT_POSE_DETECTION_CONFIDENCE, @@ -31,6 +38,9 @@ class PoseLandmarkerHelper( // If the Pose Landmarker will not change, a lazy val would be preferable. private var poseLandmarker: PoseLandmarker? = null + private var referenceCount = 0 + private var timestamp: Long = 0 + init { setupPoseLandmarker() } @@ -38,6 +48,8 @@ class PoseLandmarkerHelper( fun clearPoseLandmarker() { poseLandmarker?.close() poseLandmarker = null + referenceCount = 0 + timestamp = 0 } // Return running status of PoseLandmarkerHelper @@ -133,6 +145,9 @@ class PoseLandmarkerHelper( imageProxy: ImageProxy, isFrontCamera: Boolean ) { + // Get timestamp Value + timestamp = System.currentTimeMillis() - SystemClock.elapsedRealtime() + (imageProxy.imageInfo.timestamp / 1_000_000) + if (runningMode != RunningMode.LIVE_STREAM) { throw IllegalArgumentException( "Attempting to call detectLiveStream" + @@ -171,12 +186,45 @@ class PoseLandmarkerHelper( matrix, true ) + updateCustomVideoTrack(rotatedBitmap) + // Convert the input Bitmap object to an MPImage object to run inference val mpImage = BitmapImageBuilder(rotatedBitmap).build() detectAsync(mpImage, frameTime) } + private val handlerThread = HandlerThread("CustomVideoTrackThread").apply { start() } + private val handler = Handler(handlerThread.looper) + private var customVideoSource: CustomVideoSourceSurface? = null + + fun setCustomVideoSource(source: CustomVideoSourceSurface) { + this.customVideoSource = source + } + + private fun updateCustomVideoTrack(bitmap: Bitmap) { + handler.removeCallbacksAndMessages(null) + + handler.post { + try { + customVideoSource?.let { source -> + // Use source to draw on the canvas. + val surface = source.surfaceOrNull + if (surface == null || !surface.isValid) { + Log.e("Surface", "Surface is null or invalid") + return@post + } + + val canvas = surface.lockHardwareCanvas() + canvas.drawBitmap(bitmap, 0f, 0f, null) + surface.unlockCanvasAndPost(canvas) + } + } catch (e: Exception) { + Log.e("Surface", "Error drawing bitmap: ${e.message}") + } + } + } + // Run pose landmark using MediaPipe Pose Landmarker API @VisibleForTesting fun detectAsync(mpImage: MPImage, frameTime: Long) { @@ -192,13 +240,18 @@ class PoseLandmarkerHelper( ) { val finishTimeMs = SystemClock.uptimeMillis() val inferenceTime = finishTimeMs - result.timestampMs() + val startTimestamp = System.currentTimeMillis() + referenceCount++ poseLandmarkerHelperListener?.onResults( ResultBundle( listOf(result), inferenceTime, input.height, - input.width + input.width, + timestamp, + referenceCount, + startTimestamp ) ) } @@ -232,6 +285,9 @@ class PoseLandmarkerHelper( val inferenceTime: Long, val inputImageHeight: Int, val inputImageWidth: Int, + val presentationTimeStamp: Long, + val frameNumber: Int, + val startTimestamp: Long ) interface LandmarkerListener { diff --git a/android/src/main/java/com/tsmediapipe/fragment/CameraFragment.kt b/android/src/main/java/com/tsmediapipe/fragment/CameraFragment.kt index d459505..1fde3f7 100644 --- a/android/src/main/java/com/tsmediapipe/fragment/CameraFragment.kt +++ b/android/src/main/java/com/tsmediapipe/fragment/CameraFragment.kt @@ -22,6 +22,7 @@ import androidx.camera.lifecycle.ProcessCameraProvider import androidx.core.content.ContextCompat import androidx.fragment.app.Fragment import androidx.fragment.app.activityViewModels +import co.daily.model.customtrack.CustomVideoSourceSurface import com.facebook.react.modules.core.DeviceEventManagerModule import com.google.gson.Gson import com.google.mediapipe.tasks.vision.core.RunningMode @@ -52,11 +53,16 @@ class CameraFragment : Fragment(), PoseLandmarkerHelper.LandmarkerListener { private var imageAnalyzer: ImageAnalysis? = null private var camera: Camera? = null private var cameraProvider: ProcessCameraProvider? = null - private var cameraFacing = CameraSelector.LENS_FACING_FRONT +// private var cameraFacing = CameraSelector.LENS_FACING_FRONT + private var cameraFacing = CameraSelector.LENS_FACING_BACK /** Blocking ML operations are performed using this executor */ private lateinit var backgroundExecutor: ExecutorService + fun setCustomVideoSource(source: CustomVideoSourceSurface) { + poseLandmarkerHelper.setCustomVideoSource(source) + } + fun hasPermissions(context: Context) = PERMISSIONS_REQUIRED.all { ContextCompat.checkSelfPermission( context, @@ -298,9 +304,9 @@ class CameraFragment : Fragment(), PoseLandmarkerHelper.LandmarkerListener { val additionalData = mapOf( "height" to resultBundle.inputImageHeight, "width" to resultBundle.inputImageWidth, -// "presentationTimeStamp" to resultBundle.presentationTimeStamp, -// "frameNumber" to resultBundle.frameNumber, -// "startTimestamp" to resultBundle.startTimestamp + "presentationTimeStamp" to resultBundle.presentationTimeStamp, + "frameNumber" to resultBundle.frameNumber, + "startTimestamp" to resultBundle.startTimestamp ) val swiftDict: MutableMap = mutableMapOf( diff --git a/android/src/main/java/com/tsmediapipe/participantView/ParticipantOverlayManager.kt b/android/src/main/java/com/tsmediapipe/participantView/ParticipantOverlayManager.kt new file mode 100644 index 0000000..fb04910 --- /dev/null +++ b/android/src/main/java/com/tsmediapipe/participantView/ParticipantOverlayManager.kt @@ -0,0 +1,69 @@ +package com.tsmediapipe.participantView + +import android.annotation.SuppressLint +import android.app.Activity +import android.view.LayoutInflater +import android.view.View +import android.widget.FrameLayout +import com.tsmediapipe.R +import co.daily.model.Participant +import co.daily.model.ParticipantId +import co.daily.view.VideoView + +object ParticipantOverlayManager { + + private val videoViews = mutableMapOf() + + @SuppressLint("MissingInflatedId") + fun showOverlay(activity: Activity, participant: Participant) { + activity.runOnUiThread { + val participantContainer = activity.findViewById(R.id.participant_overlay_container) + if (participantContainer != null) { + participantContainer.visibility = View.VISIBLE + + // Inflate the participant video layout + val layoutInflater = LayoutInflater.from(activity) + val participantView = layoutInflater.inflate(R.layout.participant_view, participantContainer, false) + + // Get the VideoView from the inflated layout + val videoView = participantView.findViewById(R.id.participant_video) + + // Assign the track to the video view + videoView.track = participant.media?.camera?.track + + // Store the video view for future reference + videoViews[participant.id] = videoView + + // Add the participant view inside the overlay container + participantContainer.addView(participantView) + } + } + } + + fun hideOverlay(activity: Activity, participant: Participant) { + activity.runOnUiThread { + val participantContainer = activity.findViewById(R.id.participant_overlay_container) + val videoView = videoViews.remove(participant.id) + + if (participantContainer != null && videoView != null) { + participantContainer.removeView(videoView.parent as View) + + // Hide container if empty + if (participantContainer.childCount == 0) { + participantContainer.visibility = View.GONE + } + } + } + } + + fun participantUpdated(activity: Activity, participant: Participant) { + activity.runOnUiThread { + val videoView = videoViews[participant.id] + if (videoView != null) { + videoView.track = participant.media?.camera?.track + } else { + videoView?.track = null + } + } + } +} diff --git a/android/src/main/res/layout/fragment_my_camera.xml b/android/src/main/res/layout/fragment_my_camera.xml index 547d172..7892e9c 100644 --- a/android/src/main/res/layout/fragment_my_camera.xml +++ b/android/src/main/res/layout/fragment_my_camera.xml @@ -32,5 +32,12 @@ android:layout_width="match_parent" android:layout_height="match_parent" /> + + + diff --git a/android/src/main/res/layout/participant_view.xml b/android/src/main/res/layout/participant_view.xml new file mode 100644 index 0000000..49c66b8 --- /dev/null +++ b/android/src/main/res/layout/participant_view.xml @@ -0,0 +1,12 @@ + + + + + From 9b52e7a0de49ac57bd7b7b1986c1231a175971b8 Mon Sep 17 00:00:00 2001 From: Vishal Rabadiya Date: Tue, 13 May 2025 12:06:44 +0530 Subject: [PATCH 2/2] using front camera --- .../src/main/java/com/tsmediapipe/fragment/CameraFragment.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/android/src/main/java/com/tsmediapipe/fragment/CameraFragment.kt b/android/src/main/java/com/tsmediapipe/fragment/CameraFragment.kt index 1fde3f7..3e59ec0 100644 --- a/android/src/main/java/com/tsmediapipe/fragment/CameraFragment.kt +++ b/android/src/main/java/com/tsmediapipe/fragment/CameraFragment.kt @@ -53,8 +53,8 @@ class CameraFragment : Fragment(), PoseLandmarkerHelper.LandmarkerListener { private var imageAnalyzer: ImageAnalysis? = null private var camera: Camera? = null private var cameraProvider: ProcessCameraProvider? = null -// private var cameraFacing = CameraSelector.LENS_FACING_FRONT - private var cameraFacing = CameraSelector.LENS_FACING_BACK + private var cameraFacing = CameraSelector.LENS_FACING_FRONT + // private var cameraFacing = CameraSelector.LENS_FACING_BACK /** Blocking ML operations are performed using this executor */ private lateinit var backgroundExecutor: ExecutorService