Resolve landmark jitter by adjusting frame copy timing (#1311)

* Moved waitForEndOfFrame yield after GPU operations. For ImageReadMode.GPU, this makes sure the new texture contents are fully written before passing it on to MediaPipe. * Moved waitForEndOfFrame yield after GPU operations (for other runners).
homuler · Jan 19, 2025 · 0e04cb8 · 0e04cb8
1 parent a3e30c0
commit 0e04cb8
Show file tree

Hide file tree

Showing 6 changed files with 24 additions and 12 deletions.
diff --git a/Assets/MediaPipeUnity/Samples/Scenes/Face Detection/FaceDetectorRunner.cs b/Assets/MediaPipeUnity/Samples/Scenes/Face Detection/FaceDetectorRunner.cs
@@ -87,8 +87,6 @@ protected override IEnumerator Run()
           continue;
         }
 
-        yield return waitForEndOfFrame;
-
         // Build the input Image
         Image image;
         switch (config.ImageReadMode)
@@ -100,8 +98,12 @@ protected override IEnumerator Run()
             }
             textureFrame.ReadTextureOnGPU(imageSource.GetCurrentTexture(), flipHorizontally, flipVertically);
             image = textureFrame.BuildGPUImage(glContext);
+            // TODO: Currently we wait here for one frame to make sure the texture is fully copied to the TextureFrame before sending it to MediaPipe.
+            // This usually works but is not guaranteed. Find a proper way to do this. See: https://github.com/homuler/MediaPipeUnityPlugin/pull/1311
+            yield return waitForEndOfFrame;
             break;
           case ImageReadMode.CPU:
+            yield return waitForEndOfFrame;
             textureFrame.ReadTextureOnCPU(imageSource.GetCurrentTexture(), flipHorizontally, flipVertically);
             image = textureFrame.BuildCPUImage();
             textureFrame.Release();

diff --git a/Assets/MediaPipeUnity/Samples/Scenes/Face Landmark Detection/FaceLandmarkerRunner.cs b/Assets/MediaPipeUnity/Samples/Scenes/Face Landmark Detection/FaceLandmarkerRunner.cs
@@ -88,8 +88,6 @@ protected override IEnumerator Run()
           continue;
         }
 
-        yield return waitForEndOfFrame;
-
         // Build the input Image
         Image image;
         switch (config.ImageReadMode)
@@ -101,8 +99,12 @@ protected override IEnumerator Run()
             }
             textureFrame.ReadTextureOnGPU(imageSource.GetCurrentTexture(), flipHorizontally, flipVertically);
             image = textureFrame.BuildGPUImage(glContext);
+            // TODO: Currently we wait here for one frame to make sure the texture is fully copied to the TextureFrame before sending it to MediaPipe.
+            // This usually works but is not guaranteed. Find a proper way to do this. See: https://github.com/homuler/MediaPipeUnityPlugin/pull/1311
+            yield return waitForEndOfFrame;
             break;
           case ImageReadMode.CPU:
+            yield return waitForEndOfFrame;
             textureFrame.ReadTextureOnCPU(imageSource.GetCurrentTexture(), flipHorizontally, flipVertically);
             image = textureFrame.BuildCPUImage();
             textureFrame.Release();

diff --git a/Assets/MediaPipeUnity/Samples/Scenes/Hand Landmark Detection/HandLandmarkerRunner.cs b/Assets/MediaPipeUnity/Samples/Scenes/Hand Landmark Detection/HandLandmarkerRunner.cs
@@ -86,8 +86,6 @@ protected override IEnumerator Run()
           continue;
         }
 
-        yield return waitForEndOfFrame;
-
         // Build the input Image
         Image image;
         switch (config.ImageReadMode)
@@ -99,8 +97,12 @@ protected override IEnumerator Run()
             }
             textureFrame.ReadTextureOnGPU(imageSource.GetCurrentTexture(), flipHorizontally, flipVertically);
             image = textureFrame.BuildGPUImage(glContext);
+            // TODO: Currently we wait here for one frame to make sure the texture is fully copied to the TextureFrame before sending it to MediaPipe.
+            // This usually works but is not guaranteed. Find a proper way to do this. See: https://github.com/homuler/MediaPipeUnityPlugin/pull/1311
+            yield return waitForEndOfFrame;
             break;
           case ImageReadMode.CPU:
+            yield return waitForEndOfFrame;
             textureFrame.ReadTextureOnCPU(imageSource.GetCurrentTexture(), flipHorizontally, flipVertically);
             image = textureFrame.BuildCPUImage();
             textureFrame.Release();

diff --git a/Assets/MediaPipeUnity/Samples/Scenes/Image Segmentation/ImageSegmenterRunner.cs b/Assets/MediaPipeUnity/Samples/Scenes/Image Segmentation/ImageSegmenterRunner.cs
@@ -89,8 +89,6 @@ protected override IEnumerator Run()
           continue;
         }
 
-        yield return waitForEndOfFrame;
-
         // Build the input Image
         Image image;
         switch (config.ImageReadMode)
@@ -102,8 +100,12 @@ protected override IEnumerator Run()
             }
             textureFrame.ReadTextureOnGPU(imageSource.GetCurrentTexture(), flipHorizontally, flipVertically);
             image = textureFrame.BuildGPUImage(glContext);
+            // TODO: Currently we wait here for one frame to make sure the texture is fully copied to the TextureFrame before sending it to MediaPipe.
+            // This usually works but is not guaranteed. Find a proper way to do this. See: https://github.com/homuler/MediaPipeUnityPlugin/pull/1311
+            yield return waitForEndOfFrame;
             break;
           case ImageReadMode.CPU:
+            yield return waitForEndOfFrame;
             textureFrame.ReadTextureOnCPU(imageSource.GetCurrentTexture(), flipHorizontally, flipVertically);
             image = textureFrame.BuildCPUImage();
             textureFrame.Release();

diff --git a/Assets/MediaPipeUnity/Samples/Scenes/Object Detection/ObjectDetectorRunner.cs b/Assets/MediaPipeUnity/Samples/Scenes/Object Detection/ObjectDetectorRunner.cs
@@ -86,8 +86,6 @@ protected override IEnumerator Run()
           continue;
         }
 
-        yield return waitForEndOfFrame;
-
         // Build the input Image
         Image image;
         switch (config.ImageReadMode)
@@ -99,8 +97,12 @@ protected override IEnumerator Run()
             }
             textureFrame.ReadTextureOnGPU(imageSource.GetCurrentTexture(), flipHorizontally, flipVertically);
             image = textureFrame.BuildGPUImage(glContext);
+            // TODO: Currently we wait here for one frame to make sure the texture is fully copied to the TextureFrame before sending it to MediaPipe.
+            // This usually works but is not guaranteed. Find a proper way to do this. See: https://github.com/homuler/MediaPipeUnityPlugin/pull/1311
+            yield return waitForEndOfFrame;
             break;
           case ImageReadMode.CPU:
+            yield return waitForEndOfFrame;
             textureFrame.ReadTextureOnCPU(imageSource.GetCurrentTexture(), flipHorizontally, flipVertically);
             image = textureFrame.BuildCPUImage();
             textureFrame.Release();

diff --git a/Assets/MediaPipeUnity/Samples/Scenes/Pose Landmark Detection/PoseLandmarkerRunner.cs b/Assets/MediaPipeUnity/Samples/Scenes/Pose Landmark Detection/PoseLandmarkerRunner.cs
@@ -92,8 +92,6 @@ protected override IEnumerator Run()
           continue;
         }
 
-        yield return waitForEndOfFrame;
-
         // Build the input Image
         Image image;
         switch (config.ImageReadMode)
@@ -105,8 +103,12 @@ protected override IEnumerator Run()
             }
             textureFrame.ReadTextureOnGPU(imageSource.GetCurrentTexture(), flipHorizontally, flipVertically);
             image = textureFrame.BuildGPUImage(glContext);
+            // TODO: Currently we wait here for one frame to make sure the texture is fully copied to the TextureFrame before sending it to MediaPipe.
+            // This usually works but is not guaranteed. Find a proper way to do this. See: https://github.com/homuler/MediaPipeUnityPlugin/pull/1311
+            yield return waitForEndOfFrame;
             break;
           case ImageReadMode.CPU:
+            yield return waitForEndOfFrame;
             textureFrame.ReadTextureOnCPU(imageSource.GetCurrentTexture(), flipHorizontally, flipVertically);
             image = textureFrame.BuildCPUImage();
             textureFrame.Release();