fix(📹): Better support for rotated videos (#2461)

Author: wcandillon

docs/docs/video.md

@@ -96,7 +96,53 @@ export const useVideoFromAsset = (
 
 ## Returned Values
 
-The `useVideo` hook returns `currentFrame` which contains the current video frame, as well as `currentTime`, and `rotationInDegrees`.
+The `useVideo` hook returns `currentFrame` which contains the current video frame, as well as `currentTime`, `rotation`, and `size`.
+
+## Rotated Video
+
+`rotation` can either be `0`, `90`, `180`, or `270`.
+We provide a `fitbox` function that can help rotating and scaling the video.
+
+```tsx twoslash
+import React from "react";
+import {
+  Canvas,
+  Image,
+  useVideo,
+  fitbox,
+  rect
+} from "@shopify/react-native-skia";
+import { Pressable, useWindowDimensions } from "react-native";
+import { useSharedValue } from "react-native-reanimated";
+
+interface VideoExampleProps {
+    localVideoFile: string;
+}
+
+// The URL needs to be a local path; we usually use expo-asset for that.
+export const VideoExample = ({ localVideoFile }: VideoExampleProps) => {
+  const paused = useSharedValue(false);
+  const { width, height } = useWindowDimensions();
+  const { currentFrame, rotation, size } = useVideo(require(localVideoFile));
+  const src = rect(0, 0, size.width, size.height);
+  const dst = rect(0, 0, width, height)
+  const transform = fitbox("cover", src, dst, rotation);
+  return (
+    <Canvas style={{ flex: 1 }}>
+      <Image
+        image={currentFrame}
+        x={0}
+        y={0}
+        width={width}
+        height={height}
+        fit="none"
+        transform={transform}
+      />
+    </Canvas>
+  );
+};
+```
+
 
 ## Playback Options
 

package/android/cpp/rnskia-android/RNSkAndroidVideo.cpp

@@ -9,6 +9,7 @@
 #pragma clang diagnostic ignored "-Wdocumentation"
 
 #include "include/core/SkImage.h"
+#include "include/core/SkSize.h"
 
 #pragma clang diagnostic pop
 
@@ -102,4 +103,29 @@ float RNSkAndroidVideo::getRotationInDegrees() {
   return static_cast<float>(rotation);
 }
 
+SkISize RNSkAndroidVideo::getSize() {
+  JNIEnv *env = facebook::jni::Environment::current();
+  jclass cls = env->GetObjectClass(_jniVideo.get());
+  jmethodID mid =
+      env->GetMethodID(cls, "getSize", "()Landroid/graphics/Point;");
+  if (!mid) {
+    RNSkLogger::logToConsole("getSize method not found");
+    return SkISize::Make(0, 0);
+  }
+  jobject jPoint = env->CallObjectMethod(_jniVideo.get(), mid);
+  jclass pointCls = env->GetObjectClass(jPoint);
+
+  jfieldID xFid = env->GetFieldID(pointCls, "x", "I");
+  jfieldID yFid = env->GetFieldID(pointCls, "y", "I");
+  if (!xFid || !yFid) {
+    RNSkLogger::logToConsole("Point class fields not found");
+    return SkISize::Make(0, 0);
+  }
+
+  jint width = env->GetIntField(jPoint, xFid);
+  jint height = env->GetIntField(jPoint, yFid);
+
+  return SkISize::Make(width, height);
+}
+
 } // namespace RNSkia

package/android/cpp/rnskia-android/RNSkAndroidVideo.h

@@ -32,6 +32,7 @@ class RNSkAndroidVideo : public RNSkVideo {
   double framerate() override;
   void seek(double timestamp) override;
   float getRotationInDegrees() override;
+  SkISize getSize() override;
 };
 
 } // namespace RNSkia

package/android/src/main/java/com/shopify/reactnative/skia/RNSkVideo.java

@@ -11,6 +11,7 @@
 import android.net.Uri;
 import android.os.Build;
 import android.view.Surface;
+import android.graphics.Point;
 
 import androidx.annotation.RequiresApi;
 
@@ -30,6 +31,8 @@ public class RNSkVideo {
     private double durationMs;
     private double frameRate;
     private int rotationDegrees = 0;
+    private int width = 0;
+    private int height = 0;
 
     RNSkVideo(Context context, String localUri) {
         this.uri = Uri.parse(localUri);
@@ -57,8 +60,8 @@ private void initializeReader() {
             if (format.containsKey(MediaFormat.KEY_ROTATION)) {
                 rotationDegrees = format.getInteger(MediaFormat.KEY_ROTATION);
             }
-            int width = format.getInteger(MediaFormat.KEY_WIDTH);
-            int height = format.getInteger(MediaFormat.KEY_HEIGHT);
+            width = format.getInteger(MediaFormat.KEY_WIDTH);
+            height = format.getInteger(MediaFormat.KEY_HEIGHT);
             if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
                 imageReader = ImageReader.newInstance(
                         width,
@@ -125,6 +128,11 @@ public void seek(long timestamp) {
         }
     }
 
+    @DoNotStrip
+    public Point getSize() {
+        return new Point(width, height);
+    }
+
     private int selectVideoTrack(MediaExtractor extractor) {
         int numTracks = extractor.getTrackCount();
         for (int i = 0; i < numTracks; i++) {

package/cpp/api/JsiVideo.h

@@ -53,17 +53,27 @@ class JsiVideo : public JsiSkWrappingSharedPtrHostObject<RNSkVideo> {
     return jsi::Value::undefined();
   }
 
-  JSI_HOST_FUNCTION(getRotationInDegrees) {
+  JSI_HOST_FUNCTION(rotation) {
     auto context = getContext();
     auto rot = getObject()->getRotationInDegrees();
     return jsi::Value(static_cast<double>(rot));
   }
 
+  JSI_HOST_FUNCTION(size) {
+    auto context = getContext();
+    auto size = getObject()->getSize();
+    auto result = jsi::Object(runtime);
+    result.setProperty(runtime, "width", static_cast<double>(size.width()));
+    result.setProperty(runtime, "height", static_cast<double>(size.height()));
+    return result;
+  }
+
   JSI_EXPORT_FUNCTIONS(JSI_EXPORT_FUNC(JsiVideo, nextImage),
                        JSI_EXPORT_FUNC(JsiVideo, duration),
                        JSI_EXPORT_FUNC(JsiVideo, framerate),
                        JSI_EXPORT_FUNC(JsiVideo, seek),
-                       JSI_EXPORT_FUNC(JsiVideo, getRotationInDegrees),
+                       JSI_EXPORT_FUNC(JsiVideo, rotation),
+                       JSI_EXPORT_FUNC(JsiVideo, size),
                        JSI_EXPORT_FUNC(JsiVideo, dispose))
 
   JsiVideo(std::shared_ptr<RNSkPlatformContext> context,

package/cpp/rnskia/RNSkVideo.h

@@ -19,6 +19,7 @@ class RNSkVideo {
   virtual double framerate() = 0;
   virtual void seek(double timestamp) = 0;
   virtual float getRotationInDegrees() = 0;
+  virtual SkISize getSize() = 0;
 };
 
 } // namespace RNSkia

package/ios/RNSkia-iOS/RNSkiOSVideo.h

@@ -9,6 +9,7 @@
 #pragma clang diagnostic ignored "-Wdocumentation"
 
 #include "include/core/SkImage.h"
+#include "include/core/SkSize.h"
 
 #pragma clang diagnostic pop
 
@@ -25,6 +26,8 @@ class RNSkiOSVideo : public RNSkVideo {
   RNSkPlatformContext *_context;
   double _duration = 0;
   double _framerate = 0;
+  float _videoWidth = 0;
+  float _videoHeight = 0;
   void setupReader(CMTimeRange timeRange);
   NSDictionary *getOutputSettings();
   CGAffineTransform _preferredTransform;
@@ -37,6 +40,7 @@ class RNSkiOSVideo : public RNSkVideo {
   double framerate() override;
   void seek(double timestamp) override;
   float getRotationInDegrees() override;
+  SkISize getSize() override;
 };
 
 } // namespace RNSkia

package/ios/RNSkia-iOS/RNSkiOSVideo.mm

@@ -45,7 +45,9 @@
       [[asset tracksWithMediaType:AVMediaTypeVideo] firstObject];
   _framerate = videoTrack.nominalFrameRate;
   _preferredTransform = videoTrack.preferredTransform;
-
+  CGSize videoSize = videoTrack.naturalSize;
+  _videoWidth = videoSize.width;
+  _videoHeight = videoSize.height;
   NSDictionary *outputSettings = getOutputSettings();
   AVAssetReaderTrackOutput *trackOutput =
       [[AVAssetReaderTrackOutput alloc] initWithTrack:videoTrack
@@ -104,19 +106,15 @@
   // Determine the rotation angle in radians
   if (transform.a == 0 && transform.b == 1 && transform.c == -1 &&
       transform.d == 0) {
-    rotationAngle = M_PI_2; // 90 degrees
+    rotationAngle = 90;
   } else if (transform.a == 0 && transform.b == -1 && transform.c == 1 &&
              transform.d == 0) {
-    rotationAngle = -M_PI_2; // -90 degrees
+    rotationAngle = 270;
   } else if (transform.a == -1 && transform.b == 0 && transform.c == 0 &&
              transform.d == -1) {
-    rotationAngle = M_PI; // 180 degrees
-  } else if (transform.a == 1 && transform.b == 0 && transform.c == 0 &&
-             transform.d == 1) {
-    rotationAngle = 0.0; // 0 degrees
+    rotationAngle = 180;
   }
-  // Convert the rotation angle from radians to degrees
-  return rotationAngle * 180 / M_PI;
+  return rotationAngle;
 }
 
 void RNSkiOSVideo::seek(double timeInMilliseconds) {
@@ -136,4 +134,8 @@
 
 double RNSkiOSVideo::framerate() { return _framerate; }
 
+SkISize RNSkiOSVideo::getSize() {
+  return SkISize::Make(_videoWidth, _videoHeight);
+}
+
 } // namespace RNSkia

package/src/__tests__/snapshots/drawings/rotated-image.png

@@ -7,7 +7,10 @@ export interface Size {
   height: number;
 }
 
-export const size = (width = 0, height = 0) => ({ width, height });
+export const size = (width = 0, height = 0) => {
+  "worklet";
+  return { width, height };
+};
 
 export const rect2rect = (
   src: SkRect,
@@ -18,37 +21,19 @@ export const rect2rect = (
   { scaleX: number },
   { scaleY: number }
 ] => {
+  "worklet";
   const scaleX = dst.width / src.width;
   const scaleY = dst.height / src.height;
   const translateX = dst.x - src.x * scaleX;
   const translateY = dst.y - src.y * scaleY;
   return [{ translateX }, { translateY }, { scaleX }, { scaleY }];
 };
 
-export const fitRects = (
-  fit: Fit,
-  rect: SkRect,
-  { x, y, width, height }: SkRect
-) => {
-  const sizes = applyBoxFit(
-    fit,
-    { width: rect.width, height: rect.height },
-    { width, height }
-  );
-  const src = inscribe(sizes.src, rect);
-  const dst = inscribe(sizes.dst, {
-    x,
-    y,
-    width,
-    height,
-  });
-  return { src, dst };
-};
-
 const inscribe = (
   { width, height }: Size,
   rect: { x: number; y: number; width: number; height: number }
 ) => {
+  "worklet";
   const halfWidthDelta = (rect.width - width) / 2.0;
   const halfHeightDelta = (rect.height - height) / 2.0;
   return {
@@ -60,6 +45,7 @@ const inscribe = (
 };
 
 const applyBoxFit = (fit: Fit, input: Size, output: Size) => {
+  "worklet";
   let src = size(),
     dst = size();
   if (
@@ -122,3 +108,24 @@ const applyBoxFit = (fit: Fit, input: Size, output: Size) => {
   }
   return { src, dst };
 };
+
+export const fitRects = (
+  fit: Fit,
+  rect: SkRect,
+  { x, y, width, height }: SkRect
+) => {
+  "worklet";
+  const sizes = applyBoxFit(
+    fit,
+    { width: rect.width, height: rect.height },
+    { width, height }
+  );
+  const src = inscribe(sizes.src, rect);
+  const dst = inscribe(sizes.dst, {
+    x,
+    y,
+    width,
+    height,
+  });
+  return { src, dst };
+};

package/src/__tests__/snapshots/drawings/rotated-scaled-image.png

@@ -49,10 +49,8 @@ export const useVideo = (
   const lastTimestamp = Rea.useSharedValue(-1);
   const duration = useMemo(() => video?.duration() ?? 0, [video]);
   const framerate = useMemo(() => video?.framerate() ?? 0, [video]);
-  const rotationInDegrees = useMemo(
-    () => video?.getRotationInDegrees() ?? 0,
-    [video]
-  );
+  const size = useMemo(() => video?.size() ?? { width: 0, height: 0 }, [video]);
+  const rotation = useMemo(() => video?.rotation() ?? 0, [video]);
   Rea.useFrameCallback((frameInfo: FrameInfo) => {
     processVideoState(
       video,
@@ -78,5 +76,12 @@ export const useVideo = (
     };
   }, [video]);
 
-  return { currentFrame, currentTime, duration, framerate, rotationInDegrees };
+  return {
+    currentFrame,
+    currentTime,
+    duration,
+    framerate,
+    rotation,
+    size,
+  };
 };