From 08a1d8100e87e8418eaa3959bd1a7b1417544b39 Mon Sep 17 00:00:00 2001 From: poneciak Date: Fri, 22 Aug 2025 11:16:43 +0200 Subject: [PATCH 01/29] feat: simple worklet callback (ios only) --- apps/common-app/package.json | 7 +- .../common-app/src/examples/Record/Record.tsx | 10 +++ .../react-native-audio-api/RNAudioAPI.podspec | 5 ++ .../cpp/audioapi/AudioAPIModuleInstaller.h | 10 +-- .../HostObjects/AudioRecorderHostObject.h | 12 ++- .../audioapi/core/inputs/AudioRecorder.cpp | 39 +++++++++ .../cpp/audioapi/core/inputs/AudioRecorder.h | 17 +++- .../ios/audioapi/ios/AudioAPIModule.mm | 44 ++++++++-- packages/react-native-audio-api/package.json | 3 +- .../src/core/AudioRecorder.ts | 6 ++ .../react-native-audio-api/src/interfaces.ts | 4 + yarn.lock | 80 +++++++++++-------- 12 files changed, 187 insertions(+), 50 deletions(-) diff --git a/apps/common-app/package.json b/apps/common-app/package.json index eb9e095c1..665556058 100644 --- a/apps/common-app/package.json +++ b/apps/common-app/package.json @@ -10,15 +10,16 @@ "@react-navigation/native": "7.1.17", "@react-navigation/native-stack": "7.3.25", "@react-navigation/stack": "7.4.7", - "@shopify/react-native-skia": "2.1.1", + "@shopify/react-native-skia": "^2.1.1", "react-native-audio-api": "workspace:*", "react-native-background-timer": "^2.4.1", "react-native-dotenv": "3.4.11", "react-native-gesture-handler": "2.28.0", - "react-native-reanimated": "3.19.1", + "react-native-reanimated": "^4.0.0", "react-native-safe-area-context": "5.6.0", "react-native-screens": "4.14.1", - "react-native-svg": "15.12.1" + "react-native-svg": "15.12.1", + "react-native-worklets": "^0.4.1" }, "devDependencies": { "@babel/core": "^7.25.2", diff --git a/apps/common-app/src/examples/Record/Record.tsx b/apps/common-app/src/examples/Record/Record.tsx index 0c749aa92..7ebea7890 100644 --- a/apps/common-app/src/examples/Record/Record.tsx +++ b/apps/common-app/src/examples/Record/Record.tsx @@ -34,6 +34,16 @@ const Record: FC = () => { sampleRate: SAMPLE_RATE, bufferLengthInSamples: SAMPLE_RATE, }); + recorderRef.current.setWorkletCallback((audioData: Float32Array, timestamp: number) => { + 'worklet'; + // This runs on UI thread! + let sum = 0; + for (let i = 0; i < audioData.length; i++) { + sum += Math.abs(audioData[i]); + } + const rms = Math.sqrt(sum / audioData.length); + console.log('Audio RMS:', rms); + }); }, []); const startEcho = () => { diff --git a/packages/react-native-audio-api/RNAudioAPI.podspec b/packages/react-native-audio-api/RNAudioAPI.podspec index 2ff79cf07..8e28e1493 100644 --- a/packages/react-native-audio-api/RNAudioAPI.podspec +++ b/packages/react-native-audio-api/RNAudioAPI.podspec @@ -18,6 +18,8 @@ Pod::Spec.new do |s| s.platforms = { :ios => min_ios_version_supported } s.source = { :git => "https://github.com/software-mansion/react-native-audio-api.git", :tag => "#{s.version}" } + + s.dependency "RNWorklets" s.subspec "audioapi" do |ss| ss.source_files = "common/cpp/audioapi/**/*.{cpp,c,h}" @@ -46,9 +48,12 @@ s.pod_target_xcconfig = { "HEADER_SEARCH_PATHS" => %W[ $(PODS_TARGET_SRCROOT)/common/cpp $(PODS_TARGET_SRCROOT)/ios + #{external_dir}/include #{external_dir}/include/opus #{external_dir}/include/vorbis + $(PODS_ROOT)/Headers/Public/RNWorklets + $(PODS_ROOT)/Headers/Private/React-Core ].join(" "), 'OTHER_CFLAGS' => "$(inherited) #{folly_flags} #{fabric_flags} #{version_flag}" } diff --git a/packages/react-native-audio-api/common/cpp/audioapi/AudioAPIModuleInstaller.h b/packages/react-native-audio-api/common/cpp/audioapi/AudioAPIModuleInstaller.h index 449ebdb0f..8bd55d632 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/AudioAPIModuleInstaller.h +++ b/packages/react-native-audio-api/common/cpp/audioapi/AudioAPIModuleInstaller.h @@ -19,9 +19,9 @@ using namespace facebook; class AudioAPIModuleInstaller { public: - static void injectJSIBindings(jsi::Runtime *jsiRuntime, const std::shared_ptr &jsCallInvoker, const std::shared_ptr &audioEventHandlerRegistry) { + static void injectJSIBindings(jsi::Runtime *jsiRuntime, const std::shared_ptr &jsCallInvoker, const std::shared_ptr &audioEventHandlerRegistry, jsi::Runtime *uiRuntime = nullptr) { auto createAudioContext = getCreateAudioContextFunction(jsiRuntime, jsCallInvoker, audioEventHandlerRegistry); - auto createAudioRecorder = getCreateAudioRecorderFunction(jsiRuntime, audioEventHandlerRegistry); + auto createAudioRecorder = getCreateAudioRecorderFunction(jsiRuntime, audioEventHandlerRegistry, uiRuntime); auto createOfflineAudioContext = getCreateOfflineAudioContextFunction(jsiRuntime, jsCallInvoker, audioEventHandlerRegistry); jsiRuntime->global().setProperty(*jsiRuntime, "createAudioContext", createAudioContext); @@ -79,12 +79,12 @@ class AudioAPIModuleInstaller { }); } - static jsi::Function getCreateAudioRecorderFunction(jsi::Runtime *jsiRuntime, const std::shared_ptr &audioEventHandlerRegistry) { + static jsi::Function getCreateAudioRecorderFunction(jsi::Runtime *jsiRuntime, const std::shared_ptr &audioEventHandlerRegistry, jsi::Runtime *uiRuntime) { return jsi::Function::createFromHostFunction( *jsiRuntime, jsi::PropNameID::forAscii(*jsiRuntime, "createAudioRecorder"), 0, - [audioEventHandlerRegistry]( + [audioEventHandlerRegistry, uiRuntime]( jsi::Runtime &runtime, const jsi::Value &thisValue, const jsi::Value *args, @@ -94,7 +94,7 @@ class AudioAPIModuleInstaller { auto sampleRate = static_cast(options.getProperty(runtime, "sampleRate").getNumber()); auto bufferLength = static_cast(options.getProperty(runtime, "bufferLengthInSamples").getNumber()); - auto audioRecorderHostObject = std::make_shared(&runtime, audioEventHandlerRegistry, sampleRate, bufferLength); + auto audioRecorderHostObject = std::make_shared(&runtime, audioEventHandlerRegistry, sampleRate, bufferLength, uiRuntime); return jsi::Object::createFromHostObject(runtime, audioRecorderHostObject); }); diff --git a/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioRecorderHostObject.h b/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioRecorderHostObject.h index 207cb3da9..789174cc5 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioRecorderHostObject.h +++ b/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioRecorderHostObject.h @@ -27,7 +27,8 @@ class AudioRecorderHostObject : public JsiHostObject { jsi::Runtime *runtime, const std::shared_ptr &audioEventHandlerRegistry, float sampleRate, - int bufferLength) { + int bufferLength, + jsi::Runtime *uiRuntime) : uiRuntime_(uiRuntime) { #ifdef ANDROID audioRecorder_ = std::make_shared( sampleRate, @@ -76,11 +77,20 @@ class AudioRecorderHostObject : public JsiHostObject { return jsi::Value::undefined(); } + JSI_HOST_FUNCTION(setWorkletCallback) { + if (count > 0 && args[0].isObject() && args[0].getObject(runtime).isFunction(runtime)) { + auto callback = std::make_shared(args[0].getObject(runtime).getFunction(runtime)); + audioRecorder_->setWorkletCallback(callback, uiRuntime_); + } + return jsi::Value::undefined(); + } + JSI_PROPERTY_SETTER(onAudioReady) { audioRecorder_->setOnAudioReadyCallbackId(std::stoull(value.getString(runtime).utf8(runtime))); } private: std::shared_ptr audioRecorder_; + jsi::Runtime *uiRuntime_; }; } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/audioapi/core/inputs/AudioRecorder.cpp b/packages/react-native-audio-api/common/cpp/audioapi/core/inputs/AudioRecorder.cpp index fea55d9e7..5b296c336 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/core/inputs/AudioRecorder.cpp +++ b/packages/react-native-audio-api/common/cpp/audioapi/core/inputs/AudioRecorder.cpp @@ -22,6 +22,45 @@ AudioRecorder::AudioRecorder( isRunning_.store(false); } +void AudioRecorder::setWorkletCallback( + const std::shared_ptr &callback, + jsi::Runtime *uiRuntime) { + std::lock_guard lock(workletCallbackMutex_); + workletCallback_ = callback; + uiRuntime_ = uiRuntime; +} +void AudioRecorder::invokeWorkletOnAudioReadyCallback( + const std::shared_ptr &bus, + int numFrames, + double when) { + std::lock_guard lock(workletCallbackMutex_); + + if (!workletCallback_ || !uiRuntime_) { + return; + } + + // Get audio data from the first channel + auto channelData = bus->getChannel(0); + auto dataPtr = channelData->getData(); + auto frameCount = channelData->getSize(); + + try { + // Create JSI Array from audio data + auto jsArray = jsi::Array(*uiRuntime_, frameCount); + for (size_t i = 0; i < frameCount; i++) { + jsArray.setValueAtIndex(*uiRuntime_, i, jsi::Value(dataPtr[i])); + } + + printf("Invoking worklet callback"); + // Call the worklet directly on current thread (NAIVE - for testing only) + workletCallback_->call(*uiRuntime_, jsArray, jsi::Value(when)); + + } catch (const std::exception &e) { + // Handle error - this is important since we're on audio thread + printf("Worklet callback error: %s\n", e.what()); + } +} + void AudioRecorder::setOnAudioReadyCallbackId(uint64_t callbackId) { onAudioReadyCallbackId_ = callbackId; } diff --git a/packages/react-native-audio-api/common/cpp/audioapi/core/inputs/AudioRecorder.h b/packages/react-native-audio-api/common/cpp/audioapi/core/inputs/AudioRecorder.h index eb9843e56..3a9449b13 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/core/inputs/AudioRecorder.h +++ b/packages/react-native-audio-api/common/cpp/audioapi/core/inputs/AudioRecorder.h @@ -1,11 +1,14 @@ #pragma once +#include #include -#include #include +#include namespace audioapi { +using namespace facebook; + class RecorderAdapterNode; class AudioBus; class CircularAudioArray; @@ -40,6 +43,14 @@ class AudioRecorder { /// @note Last few frames of audio might be written to the buffer after disconnecting. void disconnect(); + + + void invokeWorkletOnAudioReadyCallback(const std::shared_ptr &bus, + int numFrames, double when); + + void setWorkletCallback(const std::shared_ptr &callback, jsi::Runtime *uiRuntime); + + virtual void start() = 0; virtual void stop() = 0; @@ -57,6 +68,10 @@ class AudioRecorder { std::shared_ptr audioEventHandlerRegistry_; uint64_t onAudioReadyCallbackId_ = 0; + std::shared_ptr workletCallback_; + jsi::Runtime *uiRuntime_; + mutable std::mutex workletCallbackMutex_; + void writeToBuffers(const float *data, int numFrames); }; diff --git a/packages/react-native-audio-api/ios/audioapi/ios/AudioAPIModule.mm b/packages/react-native-audio-api/ios/audioapi/ios/AudioAPIModule.mm index 4710eaae6..cc5f6da0a 100644 --- a/packages/react-native-audio-api/ios/audioapi/ios/AudioAPIModule.mm +++ b/packages/react-native-audio-api/ios/audioapi/ios/AudioAPIModule.mm @@ -1,9 +1,13 @@ #import #import +// #import +// #import +#import +#import #ifdef RCT_NEW_ARCH_ENABLED #import +// #import #endif // RCT_NEW_ARCH_ENABLED - #import #import #import @@ -14,6 +18,7 @@ using namespace audioapi; using namespace facebook::react; +using namespace worklets; @interface RCTBridge (JSIRuntime) - (void *)runtime; @@ -30,10 +35,20 @@ - (void)_tryAndHandleError:(dispatch_block_t)block; @implementation AudioAPIModule { std::shared_ptr _eventHandler; + std::shared_ptr workletsModuleProxy_; } #if defined(RCT_NEW_ARCH_ENABLED) @synthesize callInvoker = _callInvoker; +@synthesize moduleRegistry = _moduleRegistry; + +- (instancetype)initWithModuleRegistry:(RCTModuleRegistry *)moduleRegistry +{ + if (self = [super init]) { + _moduleRegistry = moduleRegistry; + } + return self; +} #endif // defined(RCT_NEW_ARCH_ENABLED) RCT_EXPORT_MODULE(AudioAPIModule); @@ -69,12 +84,29 @@ - (void)invalidate _eventHandler = std::make_shared(jsiRuntime, jsCallInvoker); - self.audioSessionManager = [[AudioSessionManager alloc] init]; - self.audioEngine = [[AudioEngine alloc] initWithAudioSessionManager:self.audioSessionManager]; - self.lockScreenManager = [[LockScreenManager alloc] initWithAudioAPIModule:self]; - self.notificationManager = [[NotificationManager alloc] initWithAudioAPIModule:self]; + WorkletsModule *workletsModule = [_moduleRegistry moduleForName:"WorkletsModule"]; + + if (!workletsModule) { + NSLog(@"WorkletsModule not found in module registry"); + } + + auto workletsModuleProxy = [workletsModule getWorkletsModuleProxy]; + + if (!workletsModuleProxy) { + NSLog(@"WorkletsModuleProxy not available"); + } + + workletsModuleProxy_ = workletsModuleProxy; + + auto uiWorkletRuntime = workletsModuleProxy->getUIWorkletRuntime(); + + if (!uiWorkletRuntime) { + NSLog(@"UI Worklet Runtime not available"); + } - audioapi::AudioAPIModuleInstaller::injectJSIBindings(jsiRuntime, jsCallInvoker, _eventHandler); + // Get the actual JSI Runtime reference + jsi::Runtime *uiRuntime = &(uiWorkletRuntime->getJSIRuntime()); + audioapi::AudioAPIModuleInstaller::injectJSIBindings(jsiRuntime, jsCallInvoker, _eventHandler, uiRuntime); NSLog(@"Successfully installed JSI bindings for react-native-audio-api!"); return @true; diff --git a/packages/react-native-audio-api/package.json b/packages/react-native-audio-api/package.json index da244e0ac..219637504 100644 --- a/packages/react-native-audio-api/package.json +++ b/packages/react-native-audio-api/package.json @@ -80,7 +80,8 @@ }, "peerDependencies": { "react": "*", - "react-native": "*" + "react-native": "*", + "react-native-reanimated": "^4.0.0" }, "devDependencies": { "@babel/cli": "^7.20.0", diff --git a/packages/react-native-audio-api/src/core/AudioRecorder.ts b/packages/react-native-audio-api/src/core/AudioRecorder.ts index 77eedc389..c39aa55bd 100644 --- a/packages/react-native-audio-api/src/core/AudioRecorder.ts +++ b/packages/react-native-audio-api/src/core/AudioRecorder.ts @@ -54,4 +54,10 @@ export default class AudioRecorder { this.recorder.onAudioReady = subscription.subscriptionId; } + + public setWorkletCallback( + callback: (audioData: Float32Array, timestamp: number) => void + ): void { + this.recorder.setWorkletCallback(callback); + } } diff --git a/packages/react-native-audio-api/src/interfaces.ts b/packages/react-native-audio-api/src/interfaces.ts index e0c3cf405..fc31f5976 100644 --- a/packages/react-native-audio-api/src/interfaces.ts +++ b/packages/react-native-audio-api/src/interfaces.ts @@ -203,6 +203,10 @@ export interface IAudioRecorder { // passing subscriptionId(uint_64 in cpp, string in js) to the cpp onAudioReady: string; + + setWorkletCallback: ( + callback: (audioData: Float32Array, timestamp: number) => void + ) => void; } export interface IAudioEventEmitter { diff --git a/yarn.lock b/yarn.lock index 7d0c6d834..10edf8d4f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3322,16 +3322,16 @@ __metadata: languageName: node linkType: hard -"@shopify/react-native-skia@npm:2.1.1": - version: 2.1.1 - resolution: "@shopify/react-native-skia@npm:2.1.1" +"@shopify/react-native-skia@npm:^2.1.1": + version: 2.2.4 + resolution: "@shopify/react-native-skia@npm:2.2.4" dependencies: canvaskit-wasm: "npm:0.40.0" react-reconciler: "npm:0.31.0" peerDependencies: react: ">=19.0" react-native: ">=0.78" - react-native-reanimated: ^3.0 + react-native-reanimated: ">=3.0" peerDependenciesMeta: react-native: optional: true @@ -3339,7 +3339,7 @@ __metadata: optional: true bin: setup-skia-web: scripts/setup-canvaskit.js - checksum: 10/85c4cbd313191ee59115dbdaa4ca6ca703a27184ba0d1cd950bcef49d12b47d606212f144da28a72ff356db6ae053308dab32e23c53f5294ce814b7b167b303d + checksum: 10/6262a6a4836ee3c3f4c4d8195492f2abd07e387fcc229b696a3a0851600eaa5c6930bc73738326028bf81f1feed6bc1aac952975a68100b75d8f56611b22846e languageName: node linkType: hard @@ -5068,7 +5068,7 @@ __metadata: "@react-navigation/native": "npm:7.1.17" "@react-navigation/native-stack": "npm:7.3.25" "@react-navigation/stack": "npm:7.4.7" - "@shopify/react-native-skia": "npm:2.1.1" + "@shopify/react-native-skia": "npm:^2.1.1" "@types/jest": "npm:^29.5.13" "@types/react": "npm:^19.1.0" "@types/react-native-background-timer": "npm:^2.0.2" @@ -5082,10 +5082,11 @@ __metadata: react-native-background-timer: "npm:^2.4.1" react-native-dotenv: "npm:3.4.11" react-native-gesture-handler: "npm:2.28.0" - react-native-reanimated: "npm:3.19.1" + react-native-reanimated: "npm:^4.0.0" react-native-safe-area-context: "npm:5.6.0" react-native-screens: "npm:4.14.1" react-native-svg: "npm:15.12.1" + react-native-worklets: "npm:^0.4.1" react-test-renderer: "npm:19.1.0" typescript: "npm:5.8.3" peerDependencies: @@ -11018,6 +11019,7 @@ __metadata: peerDependencies: react: "*" react-native: "*" + react-native-reanimated: ^4.0.0 bin: setup-rn-audio-api-web: ./scripts/setup-rn-audio-api-web.js languageName: unknown @@ -11089,16 +11091,6 @@ __metadata: languageName: node linkType: hard -"react-native-is-edge-to-edge@npm:1.1.7": - version: 1.1.7 - resolution: "react-native-is-edge-to-edge@npm:1.1.7" - peerDependencies: - react: "*" - react-native: "*" - checksum: 10/4cdf2b2fb5b131f2015c26d2cb7688b4a0c5f3c8474b1bf0ddfa9eabb0263df440c87262ae8f812a6ecab0d5310df0373bddad4b51f53dabb2ffee01e9ef0f44 - languageName: node - linkType: hard - "react-native-is-edge-to-edge@npm:^1.2.1": version: 1.2.1 resolution: "react-native-is-edge-to-edge@npm:1.2.1" @@ -11109,27 +11101,18 @@ __metadata: languageName: node linkType: hard -"react-native-reanimated@npm:3.19.1": - version: 3.19.1 - resolution: "react-native-reanimated@npm:3.19.1" +"react-native-reanimated@npm:^4.0.0": + version: 4.0.2 + resolution: "react-native-reanimated@npm:4.0.2" dependencies: - "@babel/plugin-transform-arrow-functions": "npm:^7.0.0-0" - "@babel/plugin-transform-class-properties": "npm:^7.0.0-0" - "@babel/plugin-transform-classes": "npm:^7.0.0-0" - "@babel/plugin-transform-nullish-coalescing-operator": "npm:^7.0.0-0" - "@babel/plugin-transform-optional-chaining": "npm:^7.0.0-0" - "@babel/plugin-transform-shorthand-properties": "npm:^7.0.0-0" - "@babel/plugin-transform-template-literals": "npm:^7.0.0-0" - "@babel/plugin-transform-unicode-regex": "npm:^7.0.0-0" - "@babel/preset-typescript": "npm:^7.16.7" - convert-source-map: "npm:^2.0.0" - invariant: "npm:^2.2.4" - react-native-is-edge-to-edge: "npm:1.1.7" + react-native-is-edge-to-edge: "npm:^1.2.1" + semver: "npm:7.7.2" peerDependencies: "@babel/core": ^7.0.0-0 react: "*" react-native: "*" - checksum: 10/2ae82f5f1dc57c50c8296b3859e0be8b31439df4eb99e8e06062af32d61fafe3d4a56b12e2044729a429afbcacbfb85287e5a854454fa4b4b21d20bc0eef7f07 + react-native-worklets: ">=0.4.0" + checksum: 10/05439bd2a7ace1a3c284efd177374ab2f2af0f9ef5a5cb793cbb963534c817ac25f6d3ba4d5372df14a04e61ad8ac0166c1307d7737e09fc747f72dead95d248 languageName: node linkType: hard @@ -11171,6 +11154,28 @@ __metadata: languageName: node linkType: hard +"react-native-worklets@npm:^0.4.1": + version: 0.4.1 + resolution: "react-native-worklets@npm:0.4.1" + dependencies: + "@babel/plugin-transform-arrow-functions": "npm:^7.0.0-0" + "@babel/plugin-transform-class-properties": "npm:^7.0.0-0" + "@babel/plugin-transform-classes": "npm:^7.0.0-0" + "@babel/plugin-transform-nullish-coalescing-operator": "npm:^7.0.0-0" + "@babel/plugin-transform-optional-chaining": "npm:^7.0.0-0" + "@babel/plugin-transform-shorthand-properties": "npm:^7.0.0-0" + "@babel/plugin-transform-template-literals": "npm:^7.0.0-0" + "@babel/plugin-transform-unicode-regex": "npm:^7.0.0-0" + "@babel/preset-typescript": "npm:^7.16.7" + convert-source-map: "npm:^2.0.0" + peerDependencies: + "@babel/core": ^7.0.0-0 + react: "*" + react-native: "*" + checksum: 10/a98da426ba8a2897d3aacf40c87d5ddda15c2511a1d246b461df7642d049e8c4ffbefbd7053823a7de81189e8b6571038c39b20b911a393566c0df7d5056b568 + languageName: node + linkType: hard + "react-native@npm:0.81.0": version: 0.81.0 resolution: "react-native@npm:0.81.0" @@ -11777,6 +11782,15 @@ __metadata: languageName: node linkType: hard +"semver@npm:7.7.2": + version: 7.7.2 + resolution: "semver@npm:7.7.2" + bin: + semver: bin/semver.js + checksum: 10/7a24cffcaa13f53c09ce55e05efe25cd41328730b2308678624f8b9f5fc3093fc4d189f47950f0b811ff8f3c3039c24a2c36717ba7961615c682045bf03e1dda + languageName: node + linkType: hard + "semver@npm:^6.3.0, semver@npm:^6.3.1": version: 6.3.1 resolution: "semver@npm:6.3.1" From 3c603f167569e20b794e93b0c94c6c582d726dab Mon Sep 17 00:00:00 2001 From: poneciak Date: Fri, 22 Aug 2025 15:57:00 +0200 Subject: [PATCH 02/29] fix: fixed worklets usage and dependencies --- apps/common-app/package.json | 2 +- .../react-native-audio-api/RNAudioAPI.podspec | 8 ++--- .../cpp/audioapi/AudioAPIModuleInstaller.h | 10 ++++-- .../HostObjects/AudioRecorderHostObject.h | 15 +++++---- .../audioapi/core/inputs/AudioRecorder.cpp | 33 +++++++------------ .../cpp/audioapi/core/inputs/AudioRecorder.h | 8 +++-- .../ios/audioapi/ios/AudioAPIModule.mm | 3 +- packages/react-native-audio-api/package.json | 3 +- .../src/core/AudioRecorder.ts | 3 +- .../react-native-audio-api/src/interfaces.ts | 5 ++- yarn.lock | 12 +++---- 11 files changed, 54 insertions(+), 48 deletions(-) diff --git a/apps/common-app/package.json b/apps/common-app/package.json index 665556058..2eb65d661 100644 --- a/apps/common-app/package.json +++ b/apps/common-app/package.json @@ -10,7 +10,7 @@ "@react-navigation/native": "7.1.17", "@react-navigation/native-stack": "7.3.25", "@react-navigation/stack": "7.4.7", - "@shopify/react-native-skia": "^2.1.1", + "@shopify/react-native-skia": "2.1.1", "react-native-audio-api": "workspace:*", "react-native-background-timer": "^2.4.1", "react-native-dotenv": "3.4.11", diff --git a/packages/react-native-audio-api/RNAudioAPI.podspec b/packages/react-native-audio-api/RNAudioAPI.podspec index 8e28e1493..b41e4e926 100644 --- a/packages/react-native-audio-api/RNAudioAPI.podspec +++ b/packages/react-native-audio-api/RNAudioAPI.podspec @@ -4,7 +4,7 @@ package_json = JSON.parse(File.read(File.join(__dir__, "package.json"))) $new_arch_enabled = ENV['RCT_NEW_ARCH_ENABLED'] == '1' -folly_flags = "-DFOLLY_NO_CONFIG -DFOLLY_MOBILE=1 -DFOLLY_USE_LIBCPP=1 -Wno-comma -Wno-shorten-64-to-32" +folly_flags = "-DFOLLY_MOBILE=1 -DFOLLY_USE_LIBCPP=1 -Wno-comma -Wno-shorten-64-to-32" fabric_flags = $new_arch_enabled ? '-DRCT_NEW_ARCH_ENABLED' : '' version_flag = "-DAUDIOAPI_VERSION=#{package_json['version']}" @@ -18,7 +18,7 @@ Pod::Spec.new do |s| s.platforms = { :ios => min_ios_version_supported } s.source = { :git => "https://github.com/software-mansion/react-native-audio-api.git", :tag => "#{s.version}" } - + s.dependency "RNWorklets" s.subspec "audioapi" do |ss| @@ -48,14 +48,14 @@ s.pod_target_xcconfig = { "HEADER_SEARCH_PATHS" => %W[ $(PODS_TARGET_SRCROOT)/common/cpp $(PODS_TARGET_SRCROOT)/ios - #{external_dir}/include #{external_dir}/include/opus #{external_dir}/include/vorbis $(PODS_ROOT)/Headers/Public/RNWorklets $(PODS_ROOT)/Headers/Private/React-Core ].join(" "), - 'OTHER_CFLAGS' => "$(inherited) #{folly_flags} #{fabric_flags} #{version_flag}" + 'OTHER_CFLAGS' => "$(inherited) #{folly_flags} #{fabric_flags} #{version_flag}", + 'OTHER_CPLUSPLUSFLAGS' => "$(inherited) #{folly_flags} #{fabric_flags} #{version_flag}" } s.user_target_xcconfig = { diff --git a/packages/react-native-audio-api/common/cpp/audioapi/AudioAPIModuleInstaller.h b/packages/react-native-audio-api/common/cpp/audioapi/AudioAPIModuleInstaller.h index 8bd55d632..70c99d2bb 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/AudioAPIModuleInstaller.h +++ b/packages/react-native-audio-api/common/cpp/audioapi/AudioAPIModuleInstaller.h @@ -11,6 +11,8 @@ #include #include +#include + #include namespace audioapi { @@ -19,7 +21,11 @@ using namespace facebook; class AudioAPIModuleInstaller { public: - static void injectJSIBindings(jsi::Runtime *jsiRuntime, const std::shared_ptr &jsCallInvoker, const std::shared_ptr &audioEventHandlerRegistry, jsi::Runtime *uiRuntime = nullptr) { + static void injectJSIBindings( + jsi::Runtime *jsiRuntime, + const std::shared_ptr &jsCallInvoker, + const std::shared_ptr &audioEventHandlerRegistry, + std::shared_ptr uiRuntime = nullptr) { auto createAudioContext = getCreateAudioContextFunction(jsiRuntime, jsCallInvoker, audioEventHandlerRegistry); auto createAudioRecorder = getCreateAudioRecorderFunction(jsiRuntime, audioEventHandlerRegistry, uiRuntime); auto createOfflineAudioContext = getCreateOfflineAudioContextFunction(jsiRuntime, jsCallInvoker, audioEventHandlerRegistry); @@ -79,7 +85,7 @@ class AudioAPIModuleInstaller { }); } - static jsi::Function getCreateAudioRecorderFunction(jsi::Runtime *jsiRuntime, const std::shared_ptr &audioEventHandlerRegistry, jsi::Runtime *uiRuntime) { + static jsi::Function getCreateAudioRecorderFunction(jsi::Runtime *jsiRuntime, const std::shared_ptr &audioEventHandlerRegistry, std::shared_ptr uiRuntime) { return jsi::Function::createFromHostFunction( *jsiRuntime, jsi::PropNameID::forAscii(*jsiRuntime, "createAudioRecorder"), diff --git a/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioRecorderHostObject.h b/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioRecorderHostObject.h index 789174cc5..384154c55 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioRecorderHostObject.h +++ b/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioRecorderHostObject.h @@ -6,6 +6,8 @@ #include #include #include +#include +#include #ifdef ANDROID #include @@ -28,7 +30,7 @@ class AudioRecorderHostObject : public JsiHostObject { const std::shared_ptr &audioEventHandlerRegistry, float sampleRate, int bufferLength, - jsi::Runtime *uiRuntime) : uiRuntime_(uiRuntime) { + std::shared_ptr uiRuntime) : uiRuntime_(uiRuntime) { #ifdef ANDROID audioRecorder_ = std::make_shared( sampleRate, @@ -49,7 +51,8 @@ class AudioRecorderHostObject : public JsiHostObject { JSI_EXPORT_FUNCTION(AudioRecorderHostObject, start), JSI_EXPORT_FUNCTION(AudioRecorderHostObject, stop), JSI_EXPORT_FUNCTION(AudioRecorderHostObject, connect), - JSI_EXPORT_FUNCTION(AudioRecorderHostObject, disconnect) + JSI_EXPORT_FUNCTION(AudioRecorderHostObject, disconnect), + JSI_EXPORT_FUNCTION(AudioRecorderHostObject, setWorkletCallback) ); } @@ -78,9 +81,9 @@ class AudioRecorderHostObject : public JsiHostObject { } JSI_HOST_FUNCTION(setWorkletCallback) { - if (count > 0 && args[0].isObject() && args[0].getObject(runtime).isFunction(runtime)) { - auto callback = std::make_shared(args[0].getObject(runtime).getFunction(runtime)); - audioRecorder_->setWorkletCallback(callback, uiRuntime_); + if (count > 0) { + auto shareableWorklet = worklets::extractShareableOrThrow(runtime, args[0]); + audioRecorder_->setWorkletCallback(shareableWorklet, uiRuntime_); } return jsi::Value::undefined(); } @@ -91,6 +94,6 @@ class AudioRecorderHostObject : public JsiHostObject { private: std::shared_ptr audioRecorder_; - jsi::Runtime *uiRuntime_; + std::shared_ptr uiRuntime_; }; } // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/audioapi/core/inputs/AudioRecorder.cpp b/packages/react-native-audio-api/common/cpp/audioapi/core/inputs/AudioRecorder.cpp index 5b296c336..22f91c499 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/core/inputs/AudioRecorder.cpp +++ b/packages/react-native-audio-api/common/cpp/audioapi/core/inputs/AudioRecorder.cpp @@ -23,10 +23,10 @@ AudioRecorder::AudioRecorder( } void AudioRecorder::setWorkletCallback( - const std::shared_ptr &callback, - jsi::Runtime *uiRuntime) { + std::shared_ptr &callback, + std::shared_ptr &uiRuntime) { std::lock_guard lock(workletCallbackMutex_); - workletCallback_ = callback; + shareableWorklet_ = callback; uiRuntime_ = uiRuntime; } void AudioRecorder::invokeWorkletOnAudioReadyCallback( @@ -35,30 +35,20 @@ void AudioRecorder::invokeWorkletOnAudioReadyCallback( double when) { std::lock_guard lock(workletCallbackMutex_); - if (!workletCallback_ || !uiRuntime_) { + if (!shareableWorklet_ || !uiRuntime_) { return; } - // Get audio data from the first channel auto channelData = bus->getChannel(0); - auto dataPtr = channelData->getData(); - auto frameCount = channelData->getSize(); - - try { - // Create JSI Array from audio data - auto jsArray = jsi::Array(*uiRuntime_, frameCount); - for (size_t i = 0; i < frameCount; i++) { - jsArray.setValueAtIndex(*uiRuntime_, i, jsi::Value(dataPtr[i])); - } - - printf("Invoking worklet callback"); - // Call the worklet directly on current thread (NAIVE - for testing only) - workletCallback_->call(*uiRuntime_, jsArray, jsi::Value(when)); - } catch (const std::exception &e) { - // Handle error - this is important since we're on audio thread - printf("Worklet callback error: %s\n", e.what()); + /// this part might throw + auto &uiRuntimeRaw = uiRuntime_->getJSIRuntime(); + auto jsArray = jsi::Array(uiRuntimeRaw, numFrames); + for (size_t i = 0; i < numFrames; i++) { + jsArray.setValueAtIndex(uiRuntimeRaw, i, jsi::Value((*channelData)[i])); } + + uiRuntime_->runGuarded(shareableWorklet_, jsArray, jsi::Value(when)); } void AudioRecorder::setOnAudioReadyCallbackId(uint64_t callbackId) { @@ -78,6 +68,7 @@ void AudioRecorder::invokeOnAudioReadyCallback( body.insert({"numFrames", numFrames}); body.insert({"when", when}); + invokeWorkletOnAudioReadyCallback(bus, numFrames, when); if (audioEventHandlerRegistry_ != nullptr) { audioEventHandlerRegistry_->invokeHandlerWithEventBody( "audioReady", onAudioReadyCallbackId_, body); diff --git a/packages/react-native-audio-api/common/cpp/audioapi/core/inputs/AudioRecorder.h b/packages/react-native-audio-api/common/cpp/audioapi/core/inputs/AudioRecorder.h index 3a9449b13..69c91ddf7 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/core/inputs/AudioRecorder.h +++ b/packages/react-native-audio-api/common/cpp/audioapi/core/inputs/AudioRecorder.h @@ -1,6 +1,8 @@ #pragma once #include +#include +#include #include #include @@ -48,7 +50,7 @@ class AudioRecorder { void invokeWorkletOnAudioReadyCallback(const std::shared_ptr &bus, int numFrames, double when); - void setWorkletCallback(const std::shared_ptr &callback, jsi::Runtime *uiRuntime); + void setWorkletCallback(std::shared_ptr &callback, std::shared_ptr &uiRuntime); virtual void start() = 0; @@ -68,8 +70,8 @@ class AudioRecorder { std::shared_ptr audioEventHandlerRegistry_; uint64_t onAudioReadyCallbackId_ = 0; - std::shared_ptr workletCallback_; - jsi::Runtime *uiRuntime_; + std::shared_ptr shareableWorklet_; + std::shared_ptr uiRuntime_; mutable std::mutex workletCallbackMutex_; void writeToBuffers(const float *data, int numFrames); diff --git a/packages/react-native-audio-api/ios/audioapi/ios/AudioAPIModule.mm b/packages/react-native-audio-api/ios/audioapi/ios/AudioAPIModule.mm index cc5f6da0a..ad359d853 100644 --- a/packages/react-native-audio-api/ios/audioapi/ios/AudioAPIModule.mm +++ b/packages/react-native-audio-api/ios/audioapi/ios/AudioAPIModule.mm @@ -105,8 +105,7 @@ - (void)invalidate } // Get the actual JSI Runtime reference - jsi::Runtime *uiRuntime = &(uiWorkletRuntime->getJSIRuntime()); - audioapi::AudioAPIModuleInstaller::injectJSIBindings(jsiRuntime, jsCallInvoker, _eventHandler, uiRuntime); + audioapi::AudioAPIModuleInstaller::injectJSIBindings(jsiRuntime, jsCallInvoker, _eventHandler, uiWorkletRuntime); NSLog(@"Successfully installed JSI bindings for react-native-audio-api!"); return @true; diff --git a/packages/react-native-audio-api/package.json b/packages/react-native-audio-api/package.json index 219637504..52f950754 100644 --- a/packages/react-native-audio-api/package.json +++ b/packages/react-native-audio-api/package.json @@ -81,7 +81,8 @@ "peerDependencies": { "react": "*", "react-native": "*", - "react-native-reanimated": "^4.0.0" + "react-native-reanimated": "^4.0.0", + "react-native-worklets": "^0.4.1" }, "devDependencies": { "@babel/cli": "^7.20.0", diff --git a/packages/react-native-audio-api/src/core/AudioRecorder.ts b/packages/react-native-audio-api/src/core/AudioRecorder.ts index c39aa55bd..97cb8bca8 100644 --- a/packages/react-native-audio-api/src/core/AudioRecorder.ts +++ b/packages/react-native-audio-api/src/core/AudioRecorder.ts @@ -4,6 +4,7 @@ import AudioBuffer from './AudioBuffer'; import { OnAudioReadyEventType } from '../events/types'; import { AudioEventEmitter } from '../events'; import RecorderAdapterNode from './RecorderAdapterNode'; +import { makeShareableCloneRecursive } from 'react-native-worklets'; export default class AudioRecorder { protected readonly recorder: IAudioRecorder; @@ -58,6 +59,6 @@ export default class AudioRecorder { public setWorkletCallback( callback: (audioData: Float32Array, timestamp: number) => void ): void { - this.recorder.setWorkletCallback(callback); + this.recorder.setWorkletCallback(makeShareableCloneRecursive(callback)); } } diff --git a/packages/react-native-audio-api/src/interfaces.ts b/packages/react-native-audio-api/src/interfaces.ts index fc31f5976..f64e29649 100644 --- a/packages/react-native-audio-api/src/interfaces.ts +++ b/packages/react-native-audio-api/src/interfaces.ts @@ -7,6 +7,7 @@ import { ChannelInterpretation, } from './types'; import { AudioEventName, AudioEventCallback } from './events/types'; +import { ShareableRef } from 'react-native-worklets'; export interface IBaseAudioContext { readonly destination: IAudioDestinationNode; @@ -205,7 +206,9 @@ export interface IAudioRecorder { onAudioReady: string; setWorkletCallback: ( - callback: (audioData: Float32Array, timestamp: number) => void + shareableRef: ShareableRef< + (audioData: Float32Array, timestamp: number) => void + > ) => void; } diff --git a/yarn.lock b/yarn.lock index 10edf8d4f..131a310a2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3322,16 +3322,16 @@ __metadata: languageName: node linkType: hard -"@shopify/react-native-skia@npm:^2.1.1": - version: 2.2.4 - resolution: "@shopify/react-native-skia@npm:2.2.4" +"@shopify/react-native-skia@npm:2.1.1": + version: 2.1.1 + resolution: "@shopify/react-native-skia@npm:2.1.1" dependencies: canvaskit-wasm: "npm:0.40.0" react-reconciler: "npm:0.31.0" peerDependencies: react: ">=19.0" react-native: ">=0.78" - react-native-reanimated: ">=3.0" + react-native-reanimated: ^3.0 peerDependenciesMeta: react-native: optional: true @@ -3339,7 +3339,7 @@ __metadata: optional: true bin: setup-skia-web: scripts/setup-canvaskit.js - checksum: 10/6262a6a4836ee3c3f4c4d8195492f2abd07e387fcc229b696a3a0851600eaa5c6930bc73738326028bf81f1feed6bc1aac952975a68100b75d8f56611b22846e + checksum: 10/85c4cbd313191ee59115dbdaa4ca6ca703a27184ba0d1cd950bcef49d12b47d606212f144da28a72ff356db6ae053308dab32e23c53f5294ce814b7b167b303d languageName: node linkType: hard @@ -5068,7 +5068,7 @@ __metadata: "@react-navigation/native": "npm:7.1.17" "@react-navigation/native-stack": "npm:7.3.25" "@react-navigation/stack": "npm:7.4.7" - "@shopify/react-native-skia": "npm:^2.1.1" + "@shopify/react-native-skia": "npm:2.1.1" "@types/jest": "npm:^29.5.13" "@types/react": "npm:^19.1.0" "@types/react-native-background-timer": "npm:^2.0.2" From 389dee3b71ed6fe3db71e8ad0846ef08933a1448 Mon Sep 17 00:00:00 2001 From: poneciak Date: Tue, 26 Aug 2025 11:00:48 +0200 Subject: [PATCH 03/29] feat: simple worklet callback (android) --- apps/fabric-example/babel.config.js | 5 +++- apps/fabric-example/package.json | 3 +- .../android/build.gradle | 28 +++++++++++++++++++ .../src/main/cpp/audioapi/CMakeLists.txt | 26 +++++++++++++++++ .../cpp/audioapi/android/AudioAPIModule.cpp | 9 ++++-- .../cpp/audioapi/android/AudioAPIModule.h | 6 ++++ .../com/swmansion/audioapi/AudioAPIModule.kt | 8 +++++- .../cpp/audioapi/AudioAPIModuleInstaller.h | 2 +- .../HostObjects/AudioRecorderHostObject.h | 4 +-- .../cpp/audioapi/core/inputs/AudioRecorder.h | 4 +-- yarn.lock | 2 ++ 11 files changed, 87 insertions(+), 10 deletions(-) diff --git a/apps/fabric-example/babel.config.js b/apps/fabric-example/babel.config.js index a2a5ea553..2a37a9250 100644 --- a/apps/fabric-example/babel.config.js +++ b/apps/fabric-example/babel.config.js @@ -2,6 +2,9 @@ module.exports = function (api) { api.cache(false); return { presets: ['module:@react-native/babel-preset'], - plugins: ['react-native-reanimated/plugin', 'module:react-native-dotenv'], + plugins: [ + 'module:react-native-dotenv', + 'react-native-worklets/plugin', + ], }; }; diff --git a/apps/fabric-example/package.json b/apps/fabric-example/package.json index 7251d0acb..e1be3e7fd 100644 --- a/apps/fabric-example/package.json +++ b/apps/fabric-example/package.json @@ -12,7 +12,8 @@ "dependencies": { "common-app": "workspace:*", "react": "19.1.0", - "react-native": "0.81.0" + "react-native": "0.81.0", + "react-native-worklets": "^0.4.1" }, "devDependencies": { "prettier": "^3.3.3" diff --git a/packages/react-native-audio-api/android/build.gradle b/packages/react-native-audio-api/android/build.gradle index 748c5ec3a..b62b65ea8 100644 --- a/packages/react-native-audio-api/android/build.gradle +++ b/packages/react-native-audio-api/android/build.gradle @@ -66,6 +66,23 @@ def resolveReactNativeDirectory() { ) } +def resolveReactNativeWorkletsDirectory() { + def reactNativeWorkletsLocation = safeAppExtGet("REACT_NATIVE_WORKLETS_NODE_MODULES_DIR", null) + if (reactNativeWorkletsLocation != null) { + return file(reactNativeWorkletsLocation) + } + + // Fallback to node resolver for custom directory structures like monorepos. + def reactNativeWorkletsPackage = file(["node", "--print", "require.resolve('react-native-worklets/package.json')"].execute(null, rootDir).text.trim()) + if (reactNativeWorkletsPackage.exists()) { + return reactNativeWorkletsPackage.parentFile + } + + throw new GradleException( + "[AudioAPI] Unable to resolve react-native-worklets location in node_modules. You should set project extension property (in `app/build.gradle`) named `REACT_NATIVE_WORKLETS_NODE_MODULES_DIR` with the path to react-native-worklets in node_modules." + ) +} + def toPlatformFileString(String path) { if (Os.isFamily(Os.FAMILY_WINDOWS)) { path = path.replace(File.separatorChar, '/' as char) @@ -83,6 +100,7 @@ static def supportsNamespace() { } def reactNativeRootDir = resolveReactNativeDirectory() +def reactNativeWorkletsRootDir = resolveReactNativeWorkletsDirectory() def reactProperties = new Properties() file("$reactNativeRootDir/ReactAndroid/gradle.properties").withInputStream { reactProperties.load(it) } @@ -118,6 +136,7 @@ android { "-DREACT_NATIVE_MINOR_VERSION=${REACT_NATIVE_MINOR_VERSION}", "-DANDROID_TOOLCHAIN=clang", "-DREACT_NATIVE_DIR=${toPlatformFileString(reactNativeRootDir.path)}", + "-DREACT_NATIVE_WORKLETS_DIR=${toPlatformFileString(reactNativeWorkletsRootDir.path)}", "-DIS_NEW_ARCHITECTURE_ENABLED=${IS_NEW_ARCHITECTURE_ENABLED}", "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON" } @@ -219,6 +238,15 @@ dependencies { implementation 'com.facebook.fbjni:fbjni:0.6.0' implementation 'com.google.oboe:oboe:1.9.3' implementation 'androidx.media:media:1.7.0' + implementation project(':react-native-worklets') +} + +// Ensure worklets is built before react-native-audio-api +evaluationDependsOn(":react-native-worklets") + +afterEvaluate { + tasks.getByName("buildCMakeDebug").dependsOn(findProject(":react-native-worklets").tasks.getByName("mergeDebugNativeLibs")) + tasks.getByName("buildCMakeRelWithDebInfo").dependsOn(findProject(":react-native-worklets").tasks.getByName("mergeReleaseNativeLibs")) } def assertMinimalReactNativeVersion = task assertMinimalReactNativeVersionTask { diff --git a/packages/react-native-audio-api/android/src/main/cpp/audioapi/CMakeLists.txt b/packages/react-native-audio-api/android/src/main/cpp/audioapi/CMakeLists.txt index 52c0cf263..5ffb6236a 100644 --- a/packages/react-native-audio-api/android/src/main/cpp/audioapi/CMakeLists.txt +++ b/packages/react-native-audio-api/android/src/main/cpp/audioapi/CMakeLists.txt @@ -18,6 +18,25 @@ find_package(ReactAndroid REQUIRED CONFIG) find_package(fbjni REQUIRED CONFIG) find_package(oboe REQUIRED CONFIG) +if(${CMAKE_BUILD_TYPE} MATCHES "Debug") + set(BUILD_TYPE "debug") +else() + set(BUILD_TYPE "release") +endif() + +# Always assume worklets is available for testing +set(WORKLETS_FOUND TRUE) +message(STATUS "Worklets support enabled - setting up imported worklets library") + +# Import the worklets library (similar to how Reanimated does it) +add_library(worklets SHARED IMPORTED) +set_target_properties( + worklets + PROPERTIES + IMPORTED_LOCATION + "${REACT_NATIVE_WORKLETS_DIR}/android/build/intermediates/cmake/${BUILD_TYPE}/obj/${ANDROID_ABI}/libworklets.so" +) + target_include_directories( react-native-audio-api PRIVATE @@ -27,8 +46,11 @@ target_include_directories( "${INCLUDE_DIR}/opus" "${INCLUDE_DIR}/vorbis" "${REACT_NATIVE_DIR}/ReactCommon" + "${REACT_NATIVE_DIR}/ReactCommon/jsiexecutor" "${REACT_NATIVE_DIR}/ReactAndroid/src/main/jni/react/turbomodule" "${REACT_NATIVE_DIR}/ReactCommon/callinvoker" + "${REACT_NATIVE_WORKLETS_DIR}/Common/cpp" + "${REACT_NATIVE_WORKLETS_DIR}/android/src/main/cpp" ) set(LINK_LIBRARIES @@ -39,6 +61,10 @@ set(LINK_LIBRARIES oboe::oboe ) +if(WORKLETS_FOUND) + list(APPEND LINK_LIBRARIES worklets) +endif() + if(ReactAndroid_VERSION_MINOR GREATER_EQUAL 76) set(RN_VERSION_LINK_LIBRARIES ReactAndroid::reactnative diff --git a/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/AudioAPIModule.cpp b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/AudioAPIModule.cpp index 7bb6f6ca5..dd0540312 100644 --- a/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/AudioAPIModule.cpp +++ b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/AudioAPIModule.cpp @@ -6,10 +6,12 @@ using namespace facebook::jni; AudioAPIModule::AudioAPIModule( jni::alias_ref &jThis, + const std::shared_ptr &workletsModuleProxy, jsi::Runtime *jsiRuntime, const std::shared_ptr &jsCallInvoker) : javaPart_(make_global(jThis)), jsiRuntime_(jsiRuntime), + workletsModuleProxy_(workletsModuleProxy), jsCallInvoker_(jsCallInvoker) { audioEventHandlerRegistry_ = std::make_shared(jsiRuntime, jsCallInvoker); @@ -17,12 +19,14 @@ AudioAPIModule::AudioAPIModule( jni::local_ref AudioAPIModule::initHybrid( jni::alias_ref jThis, + jni::alias_ref jWorkletsModule, jlong jsContext, jni::alias_ref jsCallInvokerHolder) { auto jsCallInvoker = jsCallInvokerHolder->cthis()->getCallInvoker(); auto rnRuntime = reinterpret_cast(jsContext); - return makeCxxInstance(jThis, rnRuntime, jsCallInvoker); + auto workletsModuleProxy = jWorkletsModule->cthis()->getWorkletsModuleProxy(); + return makeCxxInstance(jThis, workletsModuleProxy, rnRuntime, jsCallInvoker); } void AudioAPIModule::registerNatives() { @@ -36,8 +40,9 @@ void AudioAPIModule::registerNatives() { } void AudioAPIModule::injectJSIBindings() { + auto uiWorkletRuntime = workletsModuleProxy_->getUIWorkletRuntime(); AudioAPIModuleInstaller::injectJSIBindings( - jsiRuntime_, jsCallInvoker_, audioEventHandlerRegistry_); + jsiRuntime_, jsCallInvoker_, audioEventHandlerRegistry_, uiWorkletRuntime); } void AudioAPIModule::invokeHandlerWithEventNameAndEventBody( diff --git a/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/AudioAPIModule.h b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/AudioAPIModule.h index 04c4c134a..14c08efa0 100644 --- a/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/AudioAPIModule.h +++ b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/AudioAPIModule.h @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include #include @@ -15,6 +17,7 @@ namespace audioapi { using namespace facebook; using namespace react; +using namespace worklets; class AudioAPIModule : public jni::HybridClass { public: @@ -23,6 +26,7 @@ class AudioAPIModule : public jni::HybridClass { static jni::local_ref initHybrid( jni::alias_ref jThis, + jni::alias_ref jWorkletsModule, jlong jsContext, jni::alias_ref jsCallInvokerHolder); @@ -37,11 +41,13 @@ class AudioAPIModule : public jni::HybridClass { jni::global_ref javaPart_; jsi::Runtime *jsiRuntime_; + std::shared_ptr workletsModuleProxy_; std::shared_ptr jsCallInvoker_; std::shared_ptr audioEventHandlerRegistry_; explicit AudioAPIModule( jni::alias_ref &jThis, + const std::shared_ptr &workletsModuleProxy, jsi::Runtime *jsiRuntime, const std::shared_ptr &jsCallInvoker); }; diff --git a/packages/react-native-audio-api/android/src/main/java/com/swmansion/audioapi/AudioAPIModule.kt b/packages/react-native-audio-api/android/src/main/java/com/swmansion/audioapi/AudioAPIModule.kt index b2fa446d5..7426b4c6c 100644 --- a/packages/react-native-audio-api/android/src/main/java/com/swmansion/audioapi/AudioAPIModule.kt +++ b/packages/react-native-audio-api/android/src/main/java/com/swmansion/audioapi/AudioAPIModule.kt @@ -10,6 +10,7 @@ import com.facebook.react.module.annotations.ReactModule import com.facebook.react.turbomodule.core.CallInvokerHolderImpl import com.swmansion.audioapi.system.MediaSessionManager import com.swmansion.audioapi.system.PermissionRequestListener +import com.swmansion.worklets.WorkletsModule import java.lang.ref.WeakReference @OptIn(FrameworkAPI::class) @@ -26,6 +27,7 @@ class AudioAPIModule( private val mHybridData: HybridData external fun initHybrid( + workletsModule: WorkletsModule, jsContext: Long, callInvoker: CallInvokerHolderImpl, ): HybridData @@ -41,7 +43,11 @@ class AudioAPIModule( try { System.loadLibrary("react-native-audio-api") val jsCallInvokerHolder = reactContext.jsCallInvokerHolder as CallInvokerHolderImpl - mHybridData = initHybrid(reactContext.javaScriptContextHolder!!.get(), jsCallInvokerHolder) + + val workletsModule = reactContext.getNativeModule(WorkletsModule::class.java) + ?: throw RuntimeException("WorkletsModule not found - make sure react-native-worklets is properly installed") + + mHybridData = initHybrid(workletsModule, reactContext.javaScriptContextHolder!!.get(), jsCallInvokerHolder) } catch (exception: UnsatisfiedLinkError) { throw RuntimeException("Could not load native module AudioAPIModule", exception) } diff --git a/packages/react-native-audio-api/common/cpp/audioapi/AudioAPIModuleInstaller.h b/packages/react-native-audio-api/common/cpp/audioapi/AudioAPIModuleInstaller.h index 70c99d2bb..f9e8617c9 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/AudioAPIModuleInstaller.h +++ b/packages/react-native-audio-api/common/cpp/audioapi/AudioAPIModuleInstaller.h @@ -11,7 +11,7 @@ #include #include -#include +#include #include diff --git a/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioRecorderHostObject.h b/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioRecorderHostObject.h index 384154c55..3b4c53f9c 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioRecorderHostObject.h +++ b/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioRecorderHostObject.h @@ -6,8 +6,8 @@ #include #include #include -#include -#include +#include +#include #ifdef ANDROID #include diff --git a/packages/react-native-audio-api/common/cpp/audioapi/core/inputs/AudioRecorder.h b/packages/react-native-audio-api/common/cpp/audioapi/core/inputs/AudioRecorder.h index 69c91ddf7..05d0e7f4a 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/core/inputs/AudioRecorder.h +++ b/packages/react-native-audio-api/common/cpp/audioapi/core/inputs/AudioRecorder.h @@ -1,8 +1,8 @@ #pragma once #include -#include -#include +#include +#include #include #include diff --git a/yarn.lock b/yarn.lock index 131a310a2..4674b4203 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6625,6 +6625,7 @@ __metadata: prettier: "npm:^3.3.3" react: "npm:19.1.0" react-native: "npm:0.81.0" + react-native-worklets: "npm:^0.4.1" languageName: unknown linkType: soft @@ -11020,6 +11021,7 @@ __metadata: react: "*" react-native: "*" react-native-reanimated: ^4.0.0 + react-native-worklets: ^0.4.1 bin: setup-rn-audio-api-web: ./scripts/setup-rn-audio-api-web.js languageName: unknown From 585729dda9fcdb995fcbc2ac62e6de403b052b1f Mon Sep 17 00:00:00 2001 From: poneciak Date: Tue, 26 Aug 2025 11:04:10 +0200 Subject: [PATCH 04/29] chore: formatting and podfile.lock --- apps/fabric-example/ios/Podfile.lock | 78 ++++++++++++++----- .../cpp/audioapi/android/AudioAPIModule.cpp | 5 +- .../com/swmansion/audioapi/AudioAPIModule.kt | 9 ++- 3 files changed, 66 insertions(+), 26 deletions(-) diff --git a/apps/fabric-example/ios/Podfile.lock b/apps/fabric-example/ios/Podfile.lock index 528fa2f6b..28a0eb36f 100644 --- a/apps/fabric-example/ios/Podfile.lock +++ b/apps/fabric-example/ios/Podfile.lock @@ -2369,6 +2369,7 @@ PODS: - ReactCommon/turbomodule/bridging - ReactCommon/turbomodule/core - RNAudioAPI/audioapi (= 0.7.2) + - RNWorklets - SocketRocket - Yoga - RNAudioAPI/audioapi (0.7.2): @@ -2398,6 +2399,7 @@ PODS: - ReactCommon/turbomodule/bridging - ReactCommon/turbomodule/core - RNAudioAPI/audioapi/ios (= 0.7.2) + - RNWorklets - SocketRocket - Yoga - RNAudioAPI/audioapi/ios (0.7.2): @@ -2426,6 +2428,7 @@ PODS: - ReactCodegen - ReactCommon/turbomodule/bridging - ReactCommon/turbomodule/core + - RNWorklets - SocketRocket - Yoga - RNGestureHandler (2.28.0): @@ -2456,7 +2459,7 @@ PODS: - ReactCommon/turbomodule/core - SocketRocket - Yoga - - RNReanimated (3.19.1): + - RNReanimated (4.0.2): - boost - DoubleConversion - fast_float @@ -2483,11 +2486,11 @@ PODS: - ReactCodegen - ReactCommon/turbomodule/bridging - ReactCommon/turbomodule/core - - RNReanimated/reanimated (= 3.19.1) - - RNReanimated/worklets (= 3.19.1) + - RNReanimated/reanimated (= 4.0.2) + - RNWorklets - SocketRocket - Yoga - - RNReanimated/reanimated (3.19.1): + - RNReanimated/reanimated (4.0.2): - boost - DoubleConversion - fast_float @@ -2514,10 +2517,11 @@ PODS: - ReactCodegen - ReactCommon/turbomodule/bridging - ReactCommon/turbomodule/core - - RNReanimated/reanimated/apple (= 3.19.1) + - RNReanimated/reanimated/apple (= 4.0.2) + - RNWorklets - SocketRocket - Yoga - - RNReanimated/reanimated/apple (3.19.1): + - RNReanimated/reanimated/apple (4.0.2): - boost - DoubleConversion - fast_float @@ -2544,9 +2548,10 @@ PODS: - ReactCodegen - ReactCommon/turbomodule/bridging - ReactCommon/turbomodule/core + - RNWorklets - SocketRocket - Yoga - - RNReanimated/worklets (3.19.1): + - RNScreens (4.14.1): - boost - DoubleConversion - fast_float @@ -2562,21 +2567,21 @@ PODS: - React-Fabric - React-featureflags - React-graphics - - React-hermes - React-ImageManager - React-jsi - React-NativeModulesApple - React-RCTFabric + - React-RCTImage - React-renderercss - React-rendererdebug - React-utils - ReactCodegen - ReactCommon/turbomodule/bridging - ReactCommon/turbomodule/core - - RNReanimated/worklets/apple (= 3.19.1) + - RNScreens/common (= 4.14.1) - SocketRocket - Yoga - - RNReanimated/worklets/apple (3.19.1): + - RNScreens/common (4.14.1): - boost - DoubleConversion - fast_float @@ -2592,11 +2597,11 @@ PODS: - React-Fabric - React-featureflags - React-graphics - - React-hermes - React-ImageManager - React-jsi - React-NativeModulesApple - React-RCTFabric + - React-RCTImage - React-renderercss - React-rendererdebug - React-utils @@ -2605,7 +2610,7 @@ PODS: - ReactCommon/turbomodule/core - SocketRocket - Yoga - - RNScreens (4.14.1): + - RNSVG (15.12.1): - boost - DoubleConversion - fast_float @@ -2625,17 +2630,16 @@ PODS: - React-jsi - React-NativeModulesApple - React-RCTFabric - - React-RCTImage - React-renderercss - React-rendererdebug - React-utils - ReactCodegen - ReactCommon/turbomodule/bridging - ReactCommon/turbomodule/core - - RNScreens/common (= 4.14.1) + - RNSVG/common (= 15.12.1) - SocketRocket - Yoga - - RNScreens/common (4.14.1): + - RNSVG/common (15.12.1): - boost - DoubleConversion - fast_float @@ -2655,7 +2659,6 @@ PODS: - React-jsi - React-NativeModulesApple - React-RCTFabric - - React-RCTImage - React-renderercss - React-rendererdebug - React-utils @@ -2664,7 +2667,7 @@ PODS: - ReactCommon/turbomodule/core - SocketRocket - Yoga - - RNSVG (15.12.1): + - RNWorklets (0.4.1): - boost - DoubleConversion - fast_float @@ -2690,10 +2693,39 @@ PODS: - ReactCodegen - ReactCommon/turbomodule/bridging - ReactCommon/turbomodule/core - - RNSVG/common (= 15.12.1) + - RNWorklets/worklets (= 0.4.1) - SocketRocket - Yoga - - RNSVG/common (15.12.1): + - RNWorklets/worklets (0.4.1): + - boost + - DoubleConversion + - fast_float + - fmt + - glog + - hermes-engine + - RCT-Folly + - RCT-Folly/Fabric + - RCTRequired + - RCTTypeSafety + - React-Core + - React-debug + - React-Fabric + - React-featureflags + - React-graphics + - React-ImageManager + - React-jsi + - React-NativeModulesApple + - React-RCTFabric + - React-renderercss + - React-rendererdebug + - React-utils + - ReactCodegen + - ReactCommon/turbomodule/bridging + - ReactCommon/turbomodule/core + - RNWorklets/worklets/apple (= 0.4.1) + - SocketRocket + - Yoga + - RNWorklets/worklets/apple (0.4.1): - boost - DoubleConversion - fast_float @@ -2804,6 +2836,7 @@ DEPENDENCIES: - RNReanimated (from `../../../node_modules/react-native-reanimated`) - RNScreens (from `../../../node_modules/react-native-screens`) - RNSVG (from `../../../node_modules/react-native-svg`) + - RNWorklets (from `../../../node_modules/react-native-worklets`) - SocketRocket (~> 0.7.1) - Yoga (from `../../../node_modules/react-native/ReactCommon/yoga`) @@ -2969,6 +3002,8 @@ EXTERNAL SOURCES: :path: "../../../node_modules/react-native-screens" RNSVG: :path: "../../../node_modules/react-native-svg" + RNWorklets: + :path: "../../../node_modules/react-native-worklets" Yoga: :path: "../../../node_modules/react-native/ReactCommon/yoga" @@ -3046,11 +3081,12 @@ SPEC CHECKSUMS: ReactAppDependencyProvider: c91900fa724baee992f01c05eeb4c9e01a807f78 ReactCodegen: 8125d6ee06ea06f48f156cbddec5c2ca576d62e6 ReactCommon: 116d6ee71679243698620d8cd9a9042541e44aa6 - RNAudioAPI: 6d532d0392cf5e3a97d479a05b4be21446d6b767 + RNAudioAPI: 4b1886891b1765b334e30445993bef2491b94d9b RNGestureHandler: 3a73f098d74712952870e948b3d9cf7b6cae9961 - RNReanimated: 4e53390354d1eed1398ab51ace22b869be6ce611 + RNReanimated: b43b4178f9c5c355badd51bc0df933d1eabddc83 RNScreens: 6ced6ae8a526512a6eef6e28c2286e1fc2d378c3 RNSVG: 6f39605a4c4d200b11435c35bd077553c6b5963a + RNWorklets: 32ea3225502c57b559c76f684fc2abac865b7728 SocketRocket: d4aabe649be1e368d1318fdf28a022d714d65748 Yoga: 00013dd9cde63a2d98e8002fcc4f5ddb66c10782 diff --git a/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/AudioAPIModule.cpp b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/AudioAPIModule.cpp index dd0540312..be64a570e 100644 --- a/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/AudioAPIModule.cpp +++ b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/AudioAPIModule.cpp @@ -42,7 +42,10 @@ void AudioAPIModule::registerNatives() { void AudioAPIModule::injectJSIBindings() { auto uiWorkletRuntime = workletsModuleProxy_->getUIWorkletRuntime(); AudioAPIModuleInstaller::injectJSIBindings( - jsiRuntime_, jsCallInvoker_, audioEventHandlerRegistry_, uiWorkletRuntime); + jsiRuntime_, + jsCallInvoker_, + audioEventHandlerRegistry_, + uiWorkletRuntime); } void AudioAPIModule::invokeHandlerWithEventNameAndEventBody( diff --git a/packages/react-native-audio-api/android/src/main/java/com/swmansion/audioapi/AudioAPIModule.kt b/packages/react-native-audio-api/android/src/main/java/com/swmansion/audioapi/AudioAPIModule.kt index 7426b4c6c..7869978b3 100644 --- a/packages/react-native-audio-api/android/src/main/java/com/swmansion/audioapi/AudioAPIModule.kt +++ b/packages/react-native-audio-api/android/src/main/java/com/swmansion/audioapi/AudioAPIModule.kt @@ -43,10 +43,11 @@ class AudioAPIModule( try { System.loadLibrary("react-native-audio-api") val jsCallInvokerHolder = reactContext.jsCallInvokerHolder as CallInvokerHolderImpl - - val workletsModule = reactContext.getNativeModule(WorkletsModule::class.java) - ?: throw RuntimeException("WorkletsModule not found - make sure react-native-worklets is properly installed") - + + val workletsModule = + reactContext.getNativeModule(WorkletsModule::class.java) + ?: throw RuntimeException("WorkletsModule not found - make sure react-native-worklets is properly installed") + mHybridData = initHybrid(workletsModule, reactContext.javaScriptContextHolder!!.get(), jsCallInvokerHolder) } catch (exception: UnsatisfiedLinkError) { throw RuntimeException("Could not load native module AudioAPIModule", exception) From e143434dc1ae963d6b38d5bff858a1e94c2f9b9d Mon Sep 17 00:00:00 2001 From: poneciak Date: Tue, 26 Aug 2025 13:15:39 +0200 Subject: [PATCH 05/29] feat: created example showcasing worklets usage --- .../src/examples/Worklets/Worklets.tsx | 190 ++++++++++++++++++ apps/common-app/src/examples/index.ts | 8 + 2 files changed, 198 insertions(+) create mode 100644 apps/common-app/src/examples/Worklets/Worklets.tsx diff --git a/apps/common-app/src/examples/Worklets/Worklets.tsx b/apps/common-app/src/examples/Worklets/Worklets.tsx new file mode 100644 index 000000000..a4501569e --- /dev/null +++ b/apps/common-app/src/examples/Worklets/Worklets.tsx @@ -0,0 +1,190 @@ +import { useEffect, useRef } from "react"; +import { Text, Button, View, StyleSheet } from 'react-native'; +import { + AudioContext, + AudioManager, + AudioRecorder, + RecorderAdapterNode, +} from 'react-native-audio-api'; +import { Container } from "../../components"; +import { Extrapolation, useSharedValue } from "react-native-reanimated"; +import Animated, { + useAnimatedStyle, + withSpring, + interpolate, +} from "react-native-reanimated"; + + +function Worklets() { + const SAMPLE_RATE = 16000; + const recorderRef = useRef(null); + const aCtxRef = useRef(null); + const recorderAdapterRef = useRef(null); + + const bar0 = useSharedValue(0); + const bar1 = useSharedValue(0); + const bar2 = useSharedValue(0); // center bar + const bar3 = useSharedValue(0); + const bar4 = useSharedValue(0); + + useEffect(() => { + AudioManager.setAudioSessionOptions({ + iosCategory: 'playAndRecord', + iosMode: 'spokenAudio', + iosOptions: ['defaultToSpeaker', 'allowBluetoothA2DP'], + }); + + AudioManager.requestRecordingPermissions(); + + recorderRef.current = new AudioRecorder({ + sampleRate: SAMPLE_RATE, + bufferLengthInSamples: 512, + }); + }, []); + + const start = () => { + if (!recorderRef.current) { + console.error("Recorder is not initialized"); + return; + } + + recorderRef.current.setWorkletCallback((audioData: Float32Array, timestamp: number) => { + 'worklet'; + + // Calculates RMS amplitude + let sum = 0; + for (let i = 0; i < audioData.length; i++) { + sum += audioData[i] * audioData[i]; + } + const rms = Math.sqrt(sum / audioData.length); + const scaledAmplitude = Math.min(rms * 150, 1); + + console.log(`RMS: ${rms}, Scaled: ${scaledAmplitude}`); + bar0.value = bar1.value; + bar1.value = bar2.value; + bar3.value = bar2.value; + bar4.value = bar3.value; + + // Set new amplitude in center + bar2.value = scaledAmplitude; + }); + aCtxRef.current = new AudioContext({ sampleRate: SAMPLE_RATE }); + recorderAdapterRef.current = aCtxRef.current.createRecorderAdapter(); + recorderAdapterRef.current.connect(aCtxRef.current.destination); + recorderRef.current.connect(recorderAdapterRef.current); + recorderRef.current.start(); + console.log("Recording started"); + + if (aCtxRef.current.state === 'suspended') { + aCtxRef.current.resume(); + } + } + + const stop = () => { + if (!recorderRef.current) { + console.error("Recorder is not initialized"); + return; + } + recorderRef.current.stop(); + recorderAdapterRef.current = null; + aCtxRef.current = null; + console.log("Recording stopped"); + } + + const createBarStyle = (index: number) => { + return useAnimatedStyle(() => { + let amplitude = 0; + + switch (index) { + case 0: amplitude = bar0.value; break; + case 1: amplitude = bar1.value; break; + case 2: amplitude = bar2.value; break; + case 3: amplitude = bar3.value; break; + case 4: amplitude = bar4.value; break; + } + + const centerIndex = 2; + const distanceFromCenter = Math.abs(index - centerIndex); + + const height = interpolate( + amplitude, + [0, 1], + [10, 200], + Extrapolation.CLAMP + ); + + const backgroundColor = interpolate( + amplitude, + [0, 0.5, 1], + [0, 0.5, 1], + Extrapolation.CLAMP + ); + + const barWidth = 40 - (distanceFromCenter * 5); + const opacity = 1 - (distanceFromCenter * 0.15); + + return { + height: withSpring(height, { damping: 20, stiffness: 200 }), + width: barWidth, + backgroundColor: `rgba(${Math.floor(backgroundColor * 255)}, ${Math.floor((1 - backgroundColor) * 255)}, 100, ${opacity})`, + }; + }); + }; + + return ( + + Audio Worklets Visualizer + Speak into the microphone to see the animation + + + + {Array.from({ length: 5 }, (_, index) => ( + + ))} + + + +