fix(📸): Add audio support for videos (#2462)

Author: wcandillon

docs/docs/video.md

@@ -9,10 +9,9 @@ React Native Skia provides a way to load video frames as images, enabling rich m
 
 ## Requirements
 
+- **Reanimated** version 3 or higher.
 - **Android:** API level 26 or higher.
 - **Video URL:** Must be a local path. We recommend using it in combination with [expo-asset](https://docs.expo.dev/versions/latest/sdk/asset/) to download the video.
-- **Animated Playback:** Available only via [Reanimated 3](/docs/animations/animations) and above.
-- **Sound Playback:** Coming soon. In the meantime, audio can be played using [expo-av](https://docs.expo.dev/versions/latest/sdk/av/).
 
 ## Example
 

example/ios/Podfile.lock

@@ -356,6 +356,8 @@ PODS:
     - React
     - React-callinvoker
     - React-Core
+  - react-native-slider (4.4.2):
+    - React-Core
   - React-perflogger (0.71.7)
   - React-RCTActionSheet (0.71.7):
     - React-Core/RCTActionSheetHeaders (= 0.71.7)
@@ -511,6 +513,7 @@ DEPENDENCIES:
   - React-logger (from `../node_modules/react-native/ReactCommon/logger`)
   - react-native-safe-area-context (from `../node_modules/react-native-safe-area-context`)
   - "react-native-skia (from `../node_modules/@shopify/react-native-skia`)"
+  - "react-native-slider (from `../node_modules/@react-native-community/slider`)"
   - React-perflogger (from `../node_modules/react-native/ReactCommon/reactperflogger`)
   - React-RCTActionSheet (from `../node_modules/react-native/Libraries/ActionSheetIOS`)
   - React-RCTAnimation (from `../node_modules/react-native/Libraries/NativeAnimation`)
@@ -607,6 +610,8 @@ EXTERNAL SOURCES:
     :path: "../node_modules/react-native-safe-area-context"
   react-native-skia:
     :path: "../node_modules/@shopify/react-native-skia"
+  react-native-slider:
+    :path: "../node_modules/@react-native-community/slider"
   React-perflogger:
     :path: "../node_modules/react-native/ReactCommon/reactperflogger"
   React-RCTActionSheet:
@@ -687,6 +692,7 @@ SPEC CHECKSUMS:
   React-logger: 3f8ebad1be1bf3299d1ab6d7f971802d7395c7ef
   react-native-safe-area-context: dfe5aa13bee37a0c7e8059d14f72ffc076d120e9
   react-native-skia: c2c416b864962e73d8b9c81f0fa399ee89c8435e
+  react-native-slider: 33b8d190b59d4f67a541061bb91775d53d617d9d
   React-perflogger: 2d505bbe298e3b7bacdd9e542b15535be07220f6
   React-RCTActionSheet: 0e96e4560bd733c9b37efbf68f5b1a47615892fb
   React-RCTAnimation: fd138e26f120371c87e406745a27535e2c8a04ef

example/package.json

@@ -14,6 +14,7 @@
     "android-reverse-tcp": "adb devices | grep '\t' | awk '{print $1}' | sed 's/\\s//g' | xargs -I {} adb -s {} reverse tcp:8081 tcp:8081"
   },
   "dependencies": {
+    "@react-native-community/slider": "4.4.2",
     "@react-navigation/bottom-tabs": "6.5.7",
     "@react-navigation/elements": "1.3.6",
     "@react-navigation/native": "6.0.13",

example/src/Examples/Video/Video.tsx

@@ -4,45 +4,78 @@ import {
   ColorMatrix,
   Fill,
   ImageShader,
+  Text,
+  useFont,
 } from "@shopify/react-native-skia";
-import { Pressable, useWindowDimensions } from "react-native";
-import { useSharedValue } from "react-native-reanimated";
+import { Pressable, View, useWindowDimensions } from "react-native";
+import { useDerivedValue, useSharedValue } from "react-native-reanimated";
+import Slider from "@react-native-community/slider";
 
 import { useVideoFromAsset } from "../../components/Animations";
 
 export const Video = () => {
   const paused = useSharedValue(false);
+  const seek = useSharedValue(0);
   const { width, height } = useWindowDimensions();
-  const { currentFrame } = useVideoFromAsset(
+  const fontSize = 20;
+  const font = useFont(require("../../assets/SF-Mono-Semibold.otf"), fontSize);
+  const { currentFrame, currentTime, duration } = useVideoFromAsset(
     require("../../Tests/assets/BigBuckBunny.mp4"),
     {
       paused,
       looping: true,
+      seek,
+      volume: 0,
     }
   );
+  const text = useDerivedValue(() => currentTime.value.toFixed(0));
   return (
-    <Pressable
-      style={{ flex: 1 }}
-      onPress={() => (paused.value = !paused.value)}
-    >
-      <Canvas style={{ flex: 1 }}>
-        <Fill>
-          <ImageShader
-            image={currentFrame}
-            x={0}
-            y={0}
-            width={width}
-            height={height}
-            fit="cover"
+    <View style={{ flex: 1 }}>
+      <Pressable
+        style={{ flex: 1 }}
+        onPress={() => (paused.value = !paused.value)}
+      >
+        <Canvas style={{ flex: 1 }}>
+          <Fill>
+            <ImageShader
+              image={currentFrame}
+              x={0}
+              y={0}
+              width={width}
+              height={height}
+              fit="cover"
+            />
+            <ColorMatrix
+              matrix={[
+                0.95, 0, 0, 0, 0.05, 0.65, 0, 0, 0, 0.15, 0.15, 0, 0, 0, 0.5, 0,
+                0, 0, 1, 0,
+              ]}
+            />
+          </Fill>
+          <Text
+            x={20}
+            y={height - 200 - 2 * fontSize}
+            text={text}
+            font={font}
           />
-          <ColorMatrix
-            matrix={[
-              0.95, 0, 0, 0, 0.05, 0.65, 0, 0, 0, 0.15, 0.15, 0, 0, 0, 0.5, 0,
-              0, 0, 1, 0,
-            ]}
-          />
-        </Fill>
-      </Canvas>
-    </Pressable>
+        </Canvas>
+      </Pressable>
+      <View style={{ height: 200 }}>
+        <Slider
+          style={{ width, height: 40 }}
+          minimumValue={0}
+          maximumValue={1}
+          minimumTrackTintColor="#FFFFFF"
+          maximumTrackTintColor="#000000"
+          onSlidingComplete={(value) => {
+            seek.value = value * duration;
+            paused.value = false;
+          }}
+          onSlidingStart={() => {
+            paused.value = true;
+          }}
+        />
+      </View>
+    </View>
   );
 };

example/yarn.lock

@@ -2379,6 +2379,11 @@
   resolved "https://registry.yarnpkg.com/@react-native-community/eslint-plugin/-/eslint-plugin-1.3.0.tgz#9e558170c106bbafaa1ef502bd8e6d4651012bf9"
   integrity sha512-+zDZ20NUnSWghj7Ku5aFphMzuM9JulqCW+aPXT6IfIXFbb8tzYTTOSeRFOtuekJ99ibW2fUCSsjuKNlwDIbHFg==
 
+"@react-native-community/[email protected]":
+  version "4.4.2"
+  resolved "https://registry.yarnpkg.com/@react-native-community/slider/-/slider-4.4.2.tgz#1fea0eb3ae31841fe87bd6c4fc67569066e9cf4b"
+  integrity sha512-D9bv+3Vd2gairAhnRPAghwccgEmoM7g562pm8i4qB3Esrms5mggF81G3UvCyc0w3jjtFHh8dpQkfEoKiP0NW/Q==
+
 "@react-native/[email protected]":
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/@react-native/assets/-/assets-1.0.0.tgz#c6f9bf63d274bafc8e970628de24986b30a55c8e"

package/android/cpp/rnskia-android/RNSkAndroidVideo.cpp

@@ -83,7 +83,7 @@ double RNSkAndroidVideo::framerate() {
 void RNSkAndroidVideo::seek(double timestamp) {
   JNIEnv *env = facebook::jni::Environment::current();
   jclass cls = env->GetObjectClass(_jniVideo.get());
-  jmethodID mid = env->GetMethodID(cls, "seek", "(J)V");
+  jmethodID mid = env->GetMethodID(cls, "seek", "(D)V");
   if (!mid) {
     RNSkLogger::logToConsole("seek method not found");
     return;
@@ -128,4 +128,36 @@ SkISize RNSkAndroidVideo::getSize() {
   return SkISize::Make(width, height);
 }
 
+void RNSkAndroidVideo::play() {
+  JNIEnv *env = facebook::jni::Environment::current();
+  jclass cls = env->GetObjectClass(_jniVideo.get());
+  jmethodID mid = env->GetMethodID(cls, "play", "()V");
+  if (!mid) {
+    RNSkLogger::logToConsole("play method not found");
+    return;
+  }
+  env->CallVoidMethod(_jniVideo.get(), mid);
+}
+
+void RNSkAndroidVideo::pause() {
+  JNIEnv *env = facebook::jni::Environment::current();
+  jclass cls = env->GetObjectClass(_jniVideo.get());
+  jmethodID mid = env->GetMethodID(cls, "pause", "()V");
+  if (!mid) {
+    RNSkLogger::logToConsole("pause method not found");
+    return;
+  }
+  env->CallVoidMethod(_jniVideo.get(), mid);
+}
+
+void RNSkAndroidVideo::setVolume(float volume) {
+  JNIEnv *env = facebook::jni::Environment::current();
+  jclass cls = env->GetObjectClass(_jniVideo.get());
+  jmethodID mid = env->GetMethodID(cls, "setVolume", "(F)V");
+  if (!mid) {
+    RNSkLogger::logToConsole("setVolume method not found");
+    return;
+  }
+  env->CallVoidMethod(_jniVideo.get(), mid, volume);
+}
 } // namespace RNSkia

package/android/cpp/rnskia-android/RNSkAndroidVideo.h

@@ -33,6 +33,9 @@ class RNSkAndroidVideo : public RNSkVideo {
   void seek(double timestamp) override;
   float getRotationInDegrees() override;
   SkISize getSize() override;
+  void play() override;
+  void pause() override;
+  void setVolume(float volume) override;
 };
 
 } // namespace RNSkia

package/android/src/main/java/com/shopify/reactnative/skia/RNSkVideo.java

@@ -3,11 +3,15 @@
 import android.content.Context;
 import android.graphics.ImageFormat;
 import android.hardware.HardwareBuffer;
-import android.media.Image;
-import android.media.ImageReader;
+import android.media.AudioAttributes;
+import android.media.AudioManager;
 import android.media.MediaCodec;
 import android.media.MediaExtractor;
 import android.media.MediaFormat;
+import android.media.MediaPlayer;
+import android.media.MediaSync;
+import android.media.Image;
+import android.media.ImageReader;
 import android.net.Uri;
 import android.os.Build;
 import android.view.Surface;
@@ -28,12 +32,16 @@ public class RNSkVideo {
     private MediaCodec decoder;
     private ImageReader imageReader;
     private Surface outputSurface;
+    private MediaPlayer mediaPlayer;
+    private MediaSync mediaSync;
     private double durationMs;
     private double frameRate;
     private int rotationDegrees = 0;
     private int width = 0;
     private int height = 0;
 
+    private boolean isPlaying = false;
+
     RNSkVideo(Context context, String localUri) {
         this.uri = Uri.parse(localUri);
         this.context = context;
@@ -50,6 +58,18 @@ private void initializeReader() {
             }
             extractor.selectTrack(trackIndex);
             MediaFormat format = extractor.getTrackFormat(trackIndex);
+
+            // Initialize MediaPlayer
+            mediaPlayer = new MediaPlayer();
+            mediaPlayer.setDataSource(context, uri);
+            mediaPlayer.setAudioStreamType(AudioManager.STREAM_MUSIC);
+            mediaPlayer.setOnPreparedListener(mp -> {
+                durationMs = mp.getDuration();
+                mp.start();
+                isPlaying = true;
+            });
+            mediaPlayer.prepareAsync();
+
             // Retrieve and store video properties
             if (format.containsKey(MediaFormat.KEY_DURATION)) {
                 durationMs = format.getLong(MediaFormat.KEY_DURATION) / 1000;  // Convert microseconds to milliseconds
@@ -119,12 +139,30 @@ public HardwareBuffer nextImage() {
     }
 
     @DoNotStrip
-    public void seek(long timestamp) {
-        // Seek to the closest sync frame at or before the specified time
-        extractor.seekTo(timestamp * 1000, MediaExtractor.SEEK_TO_PREVIOUS_SYNC);
+    public void seek(double timestamp) {
+        // Log the values for debugging
+
+        long timestampUs = (long)(timestamp * 1000); // Convert milliseconds to microseconds
+
+        extractor.seekTo(timestampUs, MediaExtractor.SEEK_TO_CLOSEST_SYNC);
+        if (mediaPlayer != null) {
+            int timestampMs = (int) timestamp; // Convert to milliseconds
+            mediaPlayer.seekTo(timestampMs, MediaPlayer.SEEK_CLOSEST);
+        }
+
         // Flush the codec to reset internal state and buffers
         if (decoder != null) {
             decoder.flush();
+
+            // Decode frames until reaching the exact timestamp
+            boolean isSeeking = true;
+            while (isSeeking) {
+                decodeFrame();
+                long currentTimestampUs = extractor.getSampleTime();
+                if (currentTimestampUs >= timestampUs) {
+                    isSeeking = false;
+                }
+            }
         }
     }
 
@@ -187,7 +225,34 @@ private void decodeFrame() {
         }
     }
 
+    @DoNotStrip
+    public void play() {
+        if (mediaPlayer != null && !isPlaying) {
+            mediaPlayer.start();
+            isPlaying = true;
+        }
+    }
+
+    @DoNotStrip
+    public void pause() {
+        if (mediaPlayer != null && isPlaying) {
+            mediaPlayer.pause();
+            isPlaying = false;
+        }
+    }
+
+    @DoNotStrip
+    public void setVolume(float volume) {
+        if (mediaPlayer != null) {
+            mediaPlayer.setVolume(volume, volume);
+        }
+    }
+
     public void release() {
+        if (mediaPlayer != null) {
+            mediaPlayer.release();
+            mediaPlayer = null;
+        }
         if (decoder != null) {
             decoder.stop();
             decoder.release();

package/cpp/api/JsiVideo.h

@@ -68,13 +68,28 @@ class JsiVideo : public JsiSkWrappingSharedPtrHostObject<RNSkVideo> {
     return result;
   }
 
-  JSI_EXPORT_FUNCTIONS(JSI_EXPORT_FUNC(JsiVideo, nextImage),
-                       JSI_EXPORT_FUNC(JsiVideo, duration),
-                       JSI_EXPORT_FUNC(JsiVideo, framerate),
-                       JSI_EXPORT_FUNC(JsiVideo, seek),
-                       JSI_EXPORT_FUNC(JsiVideo, rotation),
-                       JSI_EXPORT_FUNC(JsiVideo, size),
-                       JSI_EXPORT_FUNC(JsiVideo, dispose))
+  JSI_HOST_FUNCTION(play) {
+    getObject()->play();
+    return jsi::Value::undefined();
+  }
+
+  JSI_HOST_FUNCTION(pause) {
+    getObject()->pause();
+    return jsi::Value::undefined();
+  }
+
+  JSI_HOST_FUNCTION(setVolume) {
+    auto volume = arguments[0].asNumber();
+    getObject()->setVolume(static_cast<float>(volume));
+    return jsi::Value::undefined();
+  }
+
+  JSI_EXPORT_FUNCTIONS(
+      JSI_EXPORT_FUNC(JsiVideo, nextImage), JSI_EXPORT_FUNC(JsiVideo, duration),
+      JSI_EXPORT_FUNC(JsiVideo, framerate), JSI_EXPORT_FUNC(JsiVideo, seek),
+      JSI_EXPORT_FUNC(JsiVideo, rotation), JSI_EXPORT_FUNC(JsiVideo, size),
+      JSI_EXPORT_FUNC(JsiVideo, play), JSI_EXPORT_FUNC(JsiVideo, pause),
+      JSI_EXPORT_FUNC(JsiVideo, setVolume), JSI_EXPORT_FUNC(JsiVideo, dispose))
 
   JsiVideo(std::shared_ptr<RNSkPlatformContext> context,
            std::shared_ptr<RNSkVideo> video)

package/cpp/rnskia/RNSkVideo.h

@@ -20,6 +20,9 @@ class RNSkVideo {
   virtual void seek(double timestamp) = 0;
   virtual float getRotationInDegrees() = 0;
   virtual SkISize getSize() = 0;
+  virtual void play() = 0;
+  virtual void pause() = 0;
+  virtual void setVolume(float volume) = 0;
 };
 
 } // namespace RNSkia