[blocked] feat: worklets support

apps/common-app/package.json

@@ -10,15 +10,16 @@
     "@react-navigation/native": "7.1.17",
     "@react-navigation/native-stack": "7.3.25",
     "@react-navigation/stack": "7.4.7",
-    "@shopify/react-native-skia": "2.1.1",
+    "@shopify/react-native-skia": "^2.1.1",
     "react-native-audio-api": "workspace:*",
     "react-native-background-timer": "^2.4.1",
     "react-native-dotenv": "3.4.11",
     "react-native-gesture-handler": "2.28.0",
-    "react-native-reanimated": "3.19.1",
+    "react-native-reanimated": "^4.0.0",
     "react-native-safe-area-context": "5.6.0",
     "react-native-screens": "4.14.1",
-    "react-native-svg": "15.12.1"
+    "react-native-svg": "15.12.1",
+    "react-native-worklets": "^0.4.1"
   },
   "devDependencies": {
     "@babel/core": "^7.25.2",

apps/common-app/src/examples/Record/Record.tsx

@@ -34,6 +34,16 @@ const Record: FC = () => {
       sampleRate: SAMPLE_RATE,
       bufferLengthInSamples: SAMPLE_RATE,
     });
+    recorderRef.current.setWorkletCallback((audioData: Float32Array, timestamp: number) => {
+      'worklet';
+      // This runs on UI thread!
+      let sum = 0;
+      for (let i = 0; i < audioData.length; i++) {
+        sum += Math.abs(audioData[i]);
+      }
+      const rms = Math.sqrt(sum / audioData.length);
+      console.log('Audio RMS:', rms);
+    });
   }, []);
 
   const startEcho = () => {

packages/react-native-audio-api/RNAudioAPI.podspec

@@ -18,6 +18,8 @@ Pod::Spec.new do |s|
 
   s.platforms    = { :ios => min_ios_version_supported }
   s.source       = { :git => "https://github.com/software-mansion/react-native-audio-api.git", :tag => "#{s.version}" }
+
+  s.dependency "RNWorklets"
 
   s.subspec "audioapi" do |ss|
     ss.source_files = "common/cpp/audioapi/**/*.{cpp,c,h}"
@@ -46,9 +48,12 @@ s.pod_target_xcconfig = {
   "HEADER_SEARCH_PATHS" => %W[
     $(PODS_TARGET_SRCROOT)/common/cpp
     $(PODS_TARGET_SRCROOT)/ios
+
     #{external_dir}/include
     #{external_dir}/include/opus
     #{external_dir}/include/vorbis
+    $(PODS_ROOT)/Headers/Public/RNWorklets
+    $(PODS_ROOT)/Headers/Private/React-Core
   ].join(" "),
   'OTHER_CFLAGS' => "$(inherited) #{folly_flags} #{fabric_flags} #{version_flag}"
 }

packages/react-native-audio-api/common/cpp/audioapi/AudioAPIModuleInstaller.h

@@ -19,9 +19,9 @@ using namespace facebook;
 
 class AudioAPIModuleInstaller {
  public:
-  static void injectJSIBindings(jsi::Runtime *jsiRuntime, const std::shared_ptr<react::CallInvoker> &jsCallInvoker, const std::shared_ptr<AudioEventHandlerRegistry> &audioEventHandlerRegistry) {
+  static void injectJSIBindings(jsi::Runtime *jsiRuntime, const std::shared_ptr<react::CallInvoker> &jsCallInvoker, const std::shared_ptr<AudioEventHandlerRegistry> &audioEventHandlerRegistry, jsi::Runtime *uiRuntime = nullptr) {
     auto createAudioContext = getCreateAudioContextFunction(jsiRuntime, jsCallInvoker, audioEventHandlerRegistry);
-    auto createAudioRecorder = getCreateAudioRecorderFunction(jsiRuntime, audioEventHandlerRegistry);
+    auto createAudioRecorder = getCreateAudioRecorderFunction(jsiRuntime, audioEventHandlerRegistry, uiRuntime);
     auto createOfflineAudioContext = getCreateOfflineAudioContextFunction(jsiRuntime, jsCallInvoker, audioEventHandlerRegistry);
 
     jsiRuntime->global().setProperty(*jsiRuntime, "createAudioContext", createAudioContext);
@@ -79,12 +79,12 @@ class AudioAPIModuleInstaller {
         });
   }
 
-  static jsi::Function getCreateAudioRecorderFunction(jsi::Runtime *jsiRuntime, const std::shared_ptr<AudioEventHandlerRegistry> &audioEventHandlerRegistry) {
+  static jsi::Function getCreateAudioRecorderFunction(jsi::Runtime *jsiRuntime, const std::shared_ptr<AudioEventHandlerRegistry> &audioEventHandlerRegistry, jsi::Runtime *uiRuntime) {
     return jsi::Function::createFromHostFunction(
         *jsiRuntime,
         jsi::PropNameID::forAscii(*jsiRuntime, "createAudioRecorder"),
         0,
-        [audioEventHandlerRegistry](
+        [audioEventHandlerRegistry, uiRuntime](
             jsi::Runtime &runtime,
             const jsi::Value &thisValue,
             const jsi::Value *args,
@@ -94,7 +94,7 @@ class AudioAPIModuleInstaller {
           auto sampleRate = static_cast<float>(options.getProperty(runtime, "sampleRate").getNumber());
           auto bufferLength = static_cast<int>(options.getProperty(runtime, "bufferLengthInSamples").getNumber());
 
-          auto audioRecorderHostObject = std::make_shared<AudioRecorderHostObject>(&runtime, audioEventHandlerRegistry, sampleRate, bufferLength);
+          auto audioRecorderHostObject = std::make_shared<AudioRecorderHostObject>(&runtime, audioEventHandlerRegistry, sampleRate, bufferLength, uiRuntime);
 
           return jsi::Object::createFromHostObject(runtime, audioRecorderHostObject);
         });

packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioRecorderHostObject.h

@@ -27,7 +27,8 @@ class AudioRecorderHostObject : public JsiHostObject {
       jsi::Runtime *runtime,
       const std::shared_ptr<AudioEventHandlerRegistry> &audioEventHandlerRegistry,
       float sampleRate,
-      int bufferLength) {
+      int bufferLength,
+      jsi::Runtime *uiRuntime) : uiRuntime_(uiRuntime) {
 #ifdef ANDROID
     audioRecorder_ = std::make_shared<AndroidAudioRecorder>(
       sampleRate,
@@ -76,11 +77,20 @@ class AudioRecorderHostObject : public JsiHostObject {
     return jsi::Value::undefined();
   }
 
+  JSI_HOST_FUNCTION(setWorkletCallback) {
+    if (count > 0 && args[0].isObject() && args[0].getObject(runtime).isFunction(runtime)) {
+      auto callback = std::make_shared<jsi::Function>(args[0].getObject(runtime).getFunction(runtime));
+      audioRecorder_->setWorkletCallback(callback, uiRuntime_);
+    }
+    return jsi::Value::undefined();
+  }
+
   JSI_PROPERTY_SETTER(onAudioReady) {
     audioRecorder_->setOnAudioReadyCallbackId(std::stoull(value.getString(runtime).utf8(runtime)));
   }
 
  private:
   std::shared_ptr<AudioRecorder> audioRecorder_;
+  jsi::Runtime *uiRuntime_;
 };
 } // namespace audioapi

packages/react-native-audio-api/common/cpp/audioapi/core/inputs/AudioRecorder.cpp

@@ -22,6 +22,45 @@ AudioRecorder::AudioRecorder(
   isRunning_.store(false);
 }
 
+void AudioRecorder::setWorkletCallback(
+    const std::shared_ptr<jsi::Function> &callback,
+    jsi::Runtime *uiRuntime) {
+  std::lock_guard<std::mutex> lock(workletCallbackMutex_);
+  workletCallback_ = callback;
+  uiRuntime_ = uiRuntime;
+}
+void AudioRecorder::invokeWorkletOnAudioReadyCallback(
+    const std::shared_ptr<AudioBus> &bus,
+    int numFrames,
+    double when) {
+  std::lock_guard<std::mutex> lock(workletCallbackMutex_);
+
+  if (!workletCallback_ || !uiRuntime_) {
+    return;
+  }
+
+  // Get audio data from the first channel
+  auto channelData = bus->getChannel(0);
+  auto dataPtr = channelData->getData();
+  auto frameCount = channelData->getSize();
+
+  try {
+    // Create JSI Array from audio data
+    auto jsArray = jsi::Array(*uiRuntime_, frameCount);
+    for (size_t i = 0; i < frameCount; i++) {
+      jsArray.setValueAtIndex(*uiRuntime_, i, jsi::Value(dataPtr[i]));
+    }
+
+    printf("Invoking worklet callback");
+    // Call the worklet directly on current thread (NAIVE - for testing only)
+    workletCallback_->call(*uiRuntime_, jsArray, jsi::Value(when));
+
+  } catch (const std::exception &e) {
+    // Handle error - this is important since we're on audio thread
+    printf("Worklet callback error: %s\n", e.what());
+  }
+}
+
 void AudioRecorder::setOnAudioReadyCallbackId(uint64_t callbackId) {
   onAudioReadyCallbackId_ = callbackId;
 }

packages/react-native-audio-api/common/cpp/audioapi/core/inputs/AudioRecorder.h

@@ -1,11 +1,14 @@
 #pragma once
 
+#include <jsi/jsi.h>
 
 #include <memory>
-#include <mutex>
 #include <atomic>
+#include <mutex>
 
 namespace audioapi {
+using namespace facebook;
+
 class RecorderAdapterNode;
 class AudioBus;
 class CircularAudioArray;
@@ -40,6 +43,14 @@ class AudioRecorder {
   /// @note Last few frames of audio might be written to the buffer after disconnecting.
   void disconnect();
 
+
+
+  void invokeWorkletOnAudioReadyCallback(const std::shared_ptr<AudioBus> &bus,
+                                        int numFrames, double when);
+
+  void setWorkletCallback(const std::shared_ptr<jsi::Function> &callback, jsi::Runtime *uiRuntime);
+
+
   virtual void start() = 0;
   virtual void stop() = 0;
 
@@ -57,6 +68,10 @@ class AudioRecorder {
   std::shared_ptr<AudioEventHandlerRegistry> audioEventHandlerRegistry_;
   uint64_t onAudioReadyCallbackId_ = 0;
 
+  std::shared_ptr<jsi::Function> workletCallback_;
+  jsi::Runtime *uiRuntime_;
+  mutable std::mutex workletCallbackMutex_;
+
   void writeToBuffers(const float *data, int numFrames);
 };
 

packages/react-native-audio-api/ios/audioapi/ios/AudioAPIModule.mm

@@ -1,9 +1,13 @@
 #import <React/RCTBridge+Private.h>
 #import <audioapi/ios/AudioAPIModule.h>
+// #import <react-native-worklets-core/RNWorklets.h>
+// #import <worklets/apple/WorkletsModule.h>
+#import <worklets/NativeModules/WorkletsModuleProxy.h>
+#import <worklets/apple/WorkletsModule.h>
 #ifdef RCT_NEW_ARCH_ENABLED
 #import <React/RCTCallInvoker.h>
+// #import <ReactCommon/RCTTurboModule.h>
 #endif // RCT_NEW_ARCH_ENABLED
-
 #import <audioapi/AudioAPIModuleInstaller.h>
 #import <audioapi/ios/system/AudioEngine.h>
 #import <audioapi/ios/system/AudioSessionManager.h>
@@ -14,6 +18,7 @@
 
 using namespace audioapi;
 using namespace facebook::react;
+using namespace worklets;
 
 @interface RCTBridge (JSIRuntime)
 - (void *)runtime;
@@ -30,10 +35,20 @@ - (void)_tryAndHandleError:(dispatch_block_t)block;
 
 @implementation AudioAPIModule {
   std::shared_ptr<AudioEventHandlerRegistry> _eventHandler;
+  std::shared_ptr<WorkletsModuleProxy> workletsModuleProxy_;
 }
 
 #if defined(RCT_NEW_ARCH_ENABLED)
 @synthesize callInvoker = _callInvoker;
+@synthesize moduleRegistry = _moduleRegistry;
+
+- (instancetype)initWithModuleRegistry:(RCTModuleRegistry *)moduleRegistry
+{
+  if (self = [super init]) {
+    _moduleRegistry = moduleRegistry;
+  }
+  return self;
+}
 #endif // defined(RCT_NEW_ARCH_ENABLED)
 
 RCT_EXPORT_MODULE(AudioAPIModule);
@@ -69,12 +84,29 @@ - (void)invalidate
 
   _eventHandler = std::make_shared<AudioEventHandlerRegistry>(jsiRuntime, jsCallInvoker);
 
-  self.audioSessionManager = [[AudioSessionManager alloc] init];
-  self.audioEngine = [[AudioEngine alloc] initWithAudioSessionManager:self.audioSessionManager];
-  self.lockScreenManager = [[LockScreenManager alloc] initWithAudioAPIModule:self];
-  self.notificationManager = [[NotificationManager alloc] initWithAudioAPIModule:self];
+  WorkletsModule *workletsModule = [_moduleRegistry moduleForName:"WorkletsModule"];
+
+  if (!workletsModule) {
+    NSLog(@"WorkletsModule not found in module registry");
+  }
+
+  auto workletsModuleProxy = [workletsModule getWorkletsModuleProxy];
+
+  if (!workletsModuleProxy) {
+    NSLog(@"WorkletsModuleProxy not available");
+  }
+
+  workletsModuleProxy_ = workletsModuleProxy;
+
+  auto uiWorkletRuntime = workletsModuleProxy->getUIWorkletRuntime();
+
+  if (!uiWorkletRuntime) {
+    NSLog(@"UI Worklet Runtime not available");
+  }
 
-  audioapi::AudioAPIModuleInstaller::injectJSIBindings(jsiRuntime, jsCallInvoker, _eventHandler);
+  // Get the actual JSI Runtime reference
+  jsi::Runtime *uiRuntime = &(uiWorkletRuntime->getJSIRuntime());
+  audioapi::AudioAPIModuleInstaller::injectJSIBindings(jsiRuntime, jsCallInvoker, _eventHandler, uiRuntime);
 
   NSLog(@"Successfully installed JSI bindings for react-native-audio-api!");
   return @true;

packages/react-native-audio-api/package.json

@@ -80,7 +80,8 @@
   },
   "peerDependencies": {
     "react": "*",
-    "react-native": "*"
+    "react-native": "*",
+    "react-native-reanimated": "^4.0.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.20.0",

packages/react-native-audio-api/src/core/AudioRecorder.ts

@@ -54,4 +54,10 @@ export default class AudioRecorder {
 
     this.recorder.onAudioReady = subscription.subscriptionId;
   }
+
+  public setWorkletCallback(
+    callback: (audioData: Float32Array, timestamp: number) => void
+  ): void {
+    this.recorder.setWorkletCallback(callback);
+  }
 }

packages/react-native-audio-api/src/interfaces.ts

@@ -203,6 +203,10 @@ export interface IAudioRecorder {
 
   // passing subscriptionId(uint_64 in cpp, string in js) to the cpp
   onAudioReady: string;
+
+  setWorkletCallback: (
+    callback: (audioData: Float32Array, timestamp: number) => void
+  ) => void;
 }
 
 export interface IAudioEventEmitter {