feat: Camera Passthrough

cpp/react-native-fast-opencv.cpp

@@ -41,15 +41,34 @@ jsi::Value OpenCVPlugin::get(jsi::Runtime& runtime, const jsi::PropNameID& propN
 
   if (propName == "frameBufferToMat") {
     return jsi::Function::createFromHostFunction(
-        runtime, jsi::PropNameID::forAscii(runtime, "frameBufferToMat"), 1,
+        runtime, jsi::PropNameID::forAscii(runtime, "frameBufferToMat"), 4,
         [=](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
             size_t count) -> jsi::Object {
+        double rows = arguments[0].asNumber();
+        double cols = arguments[1].asNumber();
+        int channels = arguments[2].asNumber();
+        jsi::Object input = arguments[3].asObject(runtime);
+
+        int type = -1;
+        if (channels == 1) {
+          type = CV_8U;
+        }
+        if (channels == 3) {
+          type = CV_8UC3;
+        }
+        if (channels == 4) {
+          type = CV_8UC4;
+        }
+
+        if (channels == -1) {
+          throw std::runtime_error("Fast OpenCV Error: Invalid channel count passed to frameBufferToMat!");
+        }
 
-        jsi::Object input = arguments[2].asObject(runtime);
         TypedArrayBase inputBuffer = getTypedArray(runtime, std::move(input));
         auto vec = inputBuffer.toVector(runtime);
 
-        cv::Mat mat(arguments[0].asNumber(), arguments[1].asNumber(), CV_8UC3, vec.data());
+        cv::Mat mat(rows, cols, type);
+        memcpy(mat.data, vec.data(), (int)rows * (int)cols * channels);
         auto id = FOCV_Storage::save(mat);
 
         return FOCV_JsiObject::wrap(runtime, "mat", id);

example/ios/Podfile.lock

@@ -936,7 +936,7 @@ PODS:
   - React-Mapbuffer (0.74.4):
     - glog
     - React-debug
-  - react-native-fast-opencv (0.2.0):
+  - react-native-fast-opencv (0.2.4):
     - DoubleConversion
     - glog
     - hermes-engine
@@ -979,6 +979,8 @@ PODS:
     - ReactCommon/turbomodule/bridging
     - ReactCommon/turbomodule/core
     - Yoga
+  - react-native-safe-area-context (4.11.0):
+    - React-Core
   - react-native-skia (1.3.11):
     - DoubleConversion
     - glog
@@ -1300,6 +1302,28 @@ PODS:
     - ReactCommon/turbomodule/bridging
     - ReactCommon/turbomodule/core
     - Yoga
+  - RNScreens (3.34.0):
+    - DoubleConversion
+    - glog
+    - hermes-engine
+    - RCT-Folly (= 2024.01.01.00)
+    - RCTRequired
+    - RCTTypeSafety
+    - React-Codegen
+    - React-Core
+    - React-debug
+    - React-Fabric
+    - React-featureflags
+    - React-graphics
+    - React-ImageManager
+    - React-NativeModulesApple
+    - React-RCTFabric
+    - React-RCTImage
+    - React-rendererdebug
+    - React-utils
+    - ReactCommon/turbomodule/bridging
+    - ReactCommon/turbomodule/core
+    - Yoga
   - SocketRocket (0.7.0)
   - vision-camera-resize-plugin (3.1.0):
     - DoubleConversion
@@ -1372,6 +1396,7 @@ DEPENDENCIES:
   - React-Mapbuffer (from `../node_modules/react-native/ReactCommon`)
   - react-native-fast-opencv (from `../..`)
   - react-native-image-picker (from `../node_modules/react-native-image-picker`)
+  - react-native-safe-area-context (from `../node_modules/react-native-safe-area-context`)
   - "react-native-skia (from `../node_modules/@shopify/react-native-skia`)"
   - react-native-worklets-core (from `../node_modules/react-native-worklets-core`)
   - React-nativeconfig (from `../node_modules/react-native/ReactCommon`)
@@ -1398,6 +1423,7 @@ DEPENDENCIES:
   - React-utils (from `../node_modules/react-native/ReactCommon/react/utils`)
   - ReactCommon/turbomodule/core (from `../node_modules/react-native/ReactCommon`)
   - RNReanimated (from `../node_modules/react-native-reanimated`)
+  - RNScreens (from `../node_modules/react-native-screens`)
   - vision-camera-resize-plugin (from `../node_modules/vision-camera-resize-plugin`)
   - VisionCamera (from `../node_modules/react-native-vision-camera`)
   - Yoga (from `../node_modules/react-native/ReactCommon/yoga`)
@@ -1473,6 +1499,8 @@ EXTERNAL SOURCES:
     :path: "../.."
   react-native-image-picker:
     :path: "../node_modules/react-native-image-picker"
+  react-native-safe-area-context:
+    :path: "../node_modules/react-native-safe-area-context"
   react-native-skia:
     :path: "../node_modules/@shopify/react-native-skia"
   react-native-worklets-core:
@@ -1525,6 +1553,8 @@ EXTERNAL SOURCES:
     :path: "../node_modules/react-native/ReactCommon"
   RNReanimated:
     :path: "../node_modules/react-native-reanimated"
+  RNScreens:
+    :path: "../node_modules/react-native-screens"
   vision-camera-resize-plugin:
     :path: "../node_modules/vision-camera-resize-plugin"
   VisionCamera:
@@ -1564,8 +1594,9 @@ SPEC CHECKSUMS:
   React-jsitracing: 4e9c99e73a6269b27b0d4cbab277dd90df3e5ac0
   React-logger: fbfb50e2a2b1b46ee087f0a52739fadecc5e81a4
   React-Mapbuffer: d39610dff659d8cf1fea485abae08bbf6f9c8279
-  react-native-fast-opencv: d143740f4a4f50a0946a00fbc7e8069ea5427695
+  react-native-fast-opencv: 52cb988f5dee40a599381c4617b8560977ed983c
   react-native-image-picker: c3afe5472ef870d98a4b28415fc0b928161ee5f7
+  react-native-safe-area-context: 851c62c48dce80ccaa5637b6aa5991a1bc36eca9
   react-native-skia: 8da84ea9410504bf27f0db229539a43f6caabb6a
   react-native-worklets-core: f51430dd07bf5343d4918d28a4bb00fe8f98b982
   React-nativeconfig: 2be4363c2c4ac2b42419577774e83e4e4fd2af9f
@@ -1592,10 +1623,11 @@ SPEC CHECKSUMS:
   React-utils: e8b0eac797c81c574b24f6515fec4015599b643c
   ReactCommon: eebffb37a90138c6db6eb8b2d952e7e5c6bc083c
   RNReanimated: 45553a3ae29a75a76269595f8554d07d4090e392
+  RNScreens: aa943ad421c3ced3ef5a47ede02b0cbfc43a012e
   SocketRocket: abac6f5de4d4d62d24e11868d7a2f427e0ef940d
   vision-camera-resize-plugin: 4306d5df9bce0e603bbe6ab04337f21a606f4ad1
   VisionCamera: 057aff621f7801b7d99a00d157fa39244bbd4fd2
-  Yoga: 6259f968a4fdf516d76a4432dead624d71aa0fb1
+  Yoga: 0efb3e1bd40ba59b009f01badea863281101de78
 
 PODFILE CHECKSUM: ded8a41f26047703e900afe99b8a72ca375b02ca
 

example/package.json

@@ -10,11 +10,15 @@
     "build:ios": "react-native build-ios --scheme FastOpencvExample --mode Debug --extra-params \"-sdk iphonesimulator CC=clang CPLUSPLUS=clang++ LD=clang LDPLUSPLUS=clang++ GCC_OPTIMIZATION_LEVEL=0 GCC_PRECOMPILE_PREFIX_HEADER=YES ASSETCATALOG_COMPILER_OPTIMIZATION=time DEBUG_INFORMATION_FORMAT=dwarf COMPILER_INDEX_STORE_ENABLE=NO\""
   },
   "dependencies": {
+    "@react-navigation/native": "^6.1.18",
+    "@react-navigation/native-stack": "^6.11.0",
     "@shopify/react-native-skia": "^1.3.10",
     "react": "18.2.0",
     "react-native": "0.74.4",
     "react-native-image-picker": "^7.1.2",
     "react-native-reanimated": "^3.14.0",
+    "react-native-safe-area-context": "^4.11.0",
+    "react-native-screens": "^3.34.0",
     "react-native-vision-camera": "^4.5.2",
     "react-native-worklets-core": "^1.3.3",
     "vision-camera-resize-plugin": "^3.1.0"

example/src/App.tsx

@@ -1,12 +1,32 @@
-import { ImageExample } from './ImageExample';
-import { VisionCameraExample } from './VisionCameraExample';
+import { ImageExample } from './examples/ImageExample';
+import { Route, type StackParamList } from './types';
+import { CameraRealtimeDetection } from './examples/CameraRealtimeDetection';
+import { NavigationContainer } from '@react-navigation/native';
+import { createNativeStackNavigator } from '@react-navigation/native-stack';
+import { Home } from './home/Home';
+import { CameraPassthrough } from './examples/CameraPassthrough';
 
-export default function App() {
-  const camera = false;
-
-  if (camera) {
-    return <VisionCameraExample />;
-  }
+const Stack = createNativeStackNavigator<StackParamList>();
 
-  return <ImageExample />;
+export default function App() {
+  return (
+    <NavigationContainer>
+      <Stack.Navigator>
+        <Stack.Screen
+          name={Route.Home}
+          component={Home}
+          options={{ title: 'Examples' }}
+        />
+        <Stack.Screen name={Route.ImageBlur} component={ImageExample} />
+        <Stack.Screen
+          name={Route.CameraPassthrough}
+          component={CameraPassthrough}
+        />
+        <Stack.Screen
+          name={Route.CameraRealtimeDetection}
+          component={CameraRealtimeDetection}
+        />
+      </Stack.Navigator>
+    </NavigationContainer>
+  );
 }