From cf48b0d4b767efb259a0860832810437b7b5ac06 Mon Sep 17 00:00:00 2001 From: Thomas Coldwell <31568400+thomas-coldwell@users.noreply.github.com> Date: Fri, 27 Sep 2024 09:24:35 +0100 Subject: [PATCH 1/6] feat: add warp affine transform --- cpp/FOCV_Function.cpp | 9 +- .../project.pbxproj | 4 +- example/ios/Podfile.lock | 6 +- example/src/App.tsx | 5 + .../src/examples/CameraAffineTransform.tsx | 245 ++++++++++++++++++ example/src/home/Home.tsx | 6 + example/src/types.ts | 2 + src/functions/Core.ts | 9 + 8 files changed, 280 insertions(+), 6 deletions(-) create mode 100644 example/src/examples/CameraAffineTransform.tsx diff --git a/cpp/FOCV_Function.cpp b/cpp/FOCV_Function.cpp index ba036e6..98b213c 100644 --- a/cpp/FOCV_Function.cpp +++ b/cpp/FOCV_Function.cpp @@ -743,6 +743,13 @@ jsi::Object FOCV_Function::invoke(jsi::Runtime& runtime, const jsi::Value* argum cv::transform(*src, *dst, *m); } break; + case hashString("warpAffine", 10): { + auto src = args.asMatPtr(1); + auto dst = args.asMatPtr(2); + auto m = args.asMatPtr(3); + + cv::warpAffine(*src, *dst, *m, dst->size()); + } break; case hashString("transpose", 9): { auto src = args.asMatPtr(1); auto dst = args.asMatPtr(2); @@ -1345,4 +1352,4 @@ jsi::Object FOCV_Function::invoke(jsi::Runtime& runtime, const jsi::Value* argum } return value; -} \ No newline at end of file +} diff --git a/example/ios/FastOpencvExample.xcodeproj/project.pbxproj b/example/ios/FastOpencvExample.xcodeproj/project.pbxproj index 38614c1..1221408 100644 --- a/example/ios/FastOpencvExample.xcodeproj/project.pbxproj +++ b/example/ios/FastOpencvExample.xcodeproj/project.pbxproj @@ -472,7 +472,7 @@ ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CLANG_ENABLE_MODULES = YES; CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = PCT6GL3GWD; + DEVELOPMENT_TEAM = TFP6882UUN; ENABLE_BITCODE = NO; INFOPLIST_FILE = FastOpencvExample/Info.plist; LD_RUNPATH_SEARCH_PATHS = ( @@ -500,7 +500,7 @@ ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CLANG_ENABLE_MODULES = YES; CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = PCT6GL3GWD; + DEVELOPMENT_TEAM = TFP6882UUN; INFOPLIST_FILE = FastOpencvExample/Info.plist; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index 20e351f..4fbf146 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -936,7 +936,7 @@ PODS: - React-Mapbuffer (0.74.4): - glog - React-debug - - react-native-fast-opencv (0.2.4): + - react-native-fast-opencv (0.2.6): - DoubleConversion - glog - hermes-engine @@ -1594,7 +1594,7 @@ SPEC CHECKSUMS: React-jsitracing: 4e9c99e73a6269b27b0d4cbab277dd90df3e5ac0 React-logger: fbfb50e2a2b1b46ee087f0a52739fadecc5e81a4 React-Mapbuffer: d39610dff659d8cf1fea485abae08bbf6f9c8279 - react-native-fast-opencv: 52cb988f5dee40a599381c4617b8560977ed983c + react-native-fast-opencv: 0c318e132001e3d5be0001db561e41cb331b0984 react-native-image-picker: c3afe5472ef870d98a4b28415fc0b928161ee5f7 react-native-safe-area-context: 851c62c48dce80ccaa5637b6aa5991a1bc36eca9 react-native-skia: 8da84ea9410504bf27f0db229539a43f6caabb6a @@ -1627,7 +1627,7 @@ SPEC CHECKSUMS: SocketRocket: abac6f5de4d4d62d24e11868d7a2f427e0ef940d vision-camera-resize-plugin: 4306d5df9bce0e603bbe6ab04337f21a606f4ad1 VisionCamera: 057aff621f7801b7d99a00d157fa39244bbd4fd2 - Yoga: 6259f968a4fdf516d76a4432dead624d71aa0fb1 + Yoga: 0efb3e1bd40ba59b009f01badea863281101de78 PODFILE CHECKSUM: ded8a41f26047703e900afe99b8a72ca375b02ca diff --git a/example/src/App.tsx b/example/src/App.tsx index 5fcf0dc..005a131 100644 --- a/example/src/App.tsx +++ b/example/src/App.tsx @@ -5,6 +5,7 @@ import { NavigationContainer } from '@react-navigation/native'; import { createNativeStackNavigator } from '@react-navigation/native-stack'; import { Home } from './home/Home'; import { CameraPassthrough } from './examples/CameraPassthrough'; +import { CameraAffineTransform } from './examples/CameraAffineTransform'; const Stack = createNativeStackNavigator(); @@ -26,6 +27,10 @@ export default function App() { name={Route.CameraRealtimeDetection} component={CameraRealtimeDetection} /> + ); diff --git a/example/src/examples/CameraAffineTransform.tsx b/example/src/examples/CameraAffineTransform.tsx new file mode 100644 index 0000000..2ec9e9e --- /dev/null +++ b/example/src/examples/CameraAffineTransform.tsx @@ -0,0 +1,245 @@ +import { + AlphaType, + Canvas, + ColorType, + Image, + Skia, + type SkData, + type SkImage, +} from '@shopify/react-native-skia'; +import { useEffect } from 'react'; +import { SafeAreaView, StyleSheet, Text, View } from 'react-native'; +import { DataTypes, ObjectType, OpenCV } from 'react-native-fast-opencv'; +import { useSharedValue } from 'react-native-reanimated'; +import { + Camera, + useCameraDevice, + useCameraPermission, + useFrameProcessor, +} from 'react-native-vision-camera'; +import { useRunOnJS } from 'react-native-worklets-core'; +import { useResizePlugin, type Options } from 'vision-camera-resize-plugin'; + +type PixelFormat = Options<'uint8'>['pixelFormat']; + +const WIDTH = 300; +const HEIGHT = 300; +const TARGET_TYPE = 'uint8' as const; +const TARGET_FORMAT: PixelFormat = 'rgba'; + +let lastWarn: PixelFormat | undefined; +lastWarn = undefined; +function warnNotSupported(pixelFormat: PixelFormat) { + if (lastWarn !== pixelFormat) { + console.log( + `Pixel Format '${pixelFormat}' is not natively supported by Skia! ` + + `Displaying a fall-back format that might use wrong colors instead...` + ); + lastWarn = pixelFormat; + } +} + +function getSkiaTypeForPixelFormat(pixelFormat: PixelFormat): ColorType { + switch (pixelFormat) { + case 'abgr': + case 'argb': + warnNotSupported(pixelFormat); + return ColorType.RGBA_8888; + case 'bgr': + warnNotSupported(pixelFormat); + return ColorType.RGB_888x; + case 'rgb': + return ColorType.RGB_888x; + case 'rgba': + return ColorType.RGBA_8888; + case 'bgra': + return ColorType.BGRA_8888; + } +} +function getComponentsPerPixel(pixelFormat: PixelFormat): number { + switch (pixelFormat) { + case 'abgr': + case 'rgba': + case 'bgra': + case 'argb': + return 4; + case 'rgb': + case 'bgr': + return 3; + } +} + +export function createSkiaImageFromData( + data: SkData, + width: number, + height: number, + pixelFormat: PixelFormat +): SkImage | null { + const componentsPerPixel = getComponentsPerPixel(pixelFormat); + return Skia.Image.MakeImage( + { + width: width, + height: height, + alphaType: AlphaType.Opaque, + colorType: getSkiaTypeForPixelFormat(pixelFormat), + }, + data, + width * componentsPerPixel + ); +} + +// const multiply2x3Matrix = (a: number[][], b: number[][]) => { +// 'worklet'; +// const result = []; +// for (let i = 0; i < a.length; i++) { +// const row = []; +// for (let j = 0; j < b[0].length; j++) { +// let sum = 0; +// for (let k = 0; k < a[0].length; k++) { +// sum += a[i][k] * b[k][j]; +// } +// row.push(sum); +// } +// result.push(row); +// } +// return result; +// }; + +export function CameraAffineTransform() { + const device = useCameraDevice('back'); + const { hasPermission, requestPermission } = useCameraPermission(); + const previewImage = useSharedValue(null); + const { resize } = useResizePlugin(); + + useEffect(() => { + requestPermission(); + }, [requestPermission]); + + const updatePreviewImageFromData = useRunOnJS( + (data: SkData, pixelFormat: PixelFormat) => { + const image = createSkiaImageFromData(data, WIDTH, HEIGHT, pixelFormat); + if (image == null) { + throw new Error('Failed to create Skia image from data'); + } + previewImage.value?.dispose(); + previewImage.value = image; + }, + [] + ); + + const frameProcessor = useFrameProcessor( + (frame) => { + 'worklet'; + + const resized = resize(frame, { + scale: { + width: WIDTH, + height: HEIGHT, + }, + crop: { + x: 0, + y: 0, + width: frame.width, + height: frame.height, + }, + pixelFormat: TARGET_FORMAT, + dataType: TARGET_TYPE, + rotation: '90deg', + }); + + const frameMat = OpenCV.frameBufferToMat(HEIGHT, WIDTH, 4, resized); + const outputMat = OpenCV.createObject( + ObjectType.Mat, + HEIGHT, + WIDTH, + DataTypes.CV_8UC4 + ); + + const scaleX = frame.height / frame.width; + const scaledFrameWidth = WIDTH * scaleX; + const translateX = (scaledFrameWidth - WIDTH) / 2; + + // 2x3 affine matrix has the following form (for scale and translation only): + // | sx 0 tx | + // | 0 sy ty | + const matrix = [ + [scaleX, 0.0, -translateX], + [0.0, 1.0, 0.0], + ]; + + const transformMat = OpenCV.createObject( + ObjectType.Mat, + 2, + 3, + DataTypes.CV_32F, + matrix.flat() + ); + + OpenCV.invoke('warpAffine', frameMat, outputMat, transformMat); + + const output = OpenCV.matToBuffer(outputMat, 'uint8'); + const data = Skia.Data.fromBytes(output.buffer); + + updatePreviewImageFromData(data, TARGET_FORMAT).then(() => { + data.dispose(); + }); + + OpenCV.clearBuffers(); // REMEMBER TO CLEAN + }, + [updatePreviewImageFromData] + ); + + if (!hasPermission) { + return No permission; + } + + if (device == null) { + return No device; + } + + return ( + + + + + + + + + ); +} + +const styles = StyleSheet.create({ + container: { + flex: 1, + width: '100%', + alignItems: 'center', + justifyContent: 'center', + }, + safe: { + flex: 1, + width: '100%', + alignItems: 'center', + justifyContent: 'flex-end', + }, + canvas: { + width: WIDTH, + height: HEIGHT, + borderColor: 'red', + borderWidth: 2, + }, +}); diff --git a/example/src/home/Home.tsx b/example/src/home/Home.tsx index e3e41c7..7cbaf9a 100644 --- a/example/src/home/Home.tsx +++ b/example/src/home/Home.tsx @@ -29,6 +29,12 @@ const items: ItemDetails[] = [ description: 'Detect objects in realtime', route: Route.CameraRealtimeDetection, }, + { + title: 'Camera Affine Transform', + emoji: '📷', + description: 'Apply affine transform to camera feed', + route: Route.CameraAffineTransform, + }, ]; export const Home = () => { diff --git a/example/src/types.ts b/example/src/types.ts index f9229e4..581b120 100644 --- a/example/src/types.ts +++ b/example/src/types.ts @@ -5,6 +5,7 @@ export enum Route { ImageBlur = 'ImageBlur', CameraPassthrough = 'CameraPassthrough', CameraRealtimeDetection = 'CameraRealtimeDetection', + CameraAffineTransform = 'CameraAffineTransform', } export interface StackParamList extends ParamListBase { @@ -12,6 +13,7 @@ export interface StackParamList extends ParamListBase { ImageBlur: undefined; CameraPassthrough: undefined; CameraRealtimeDetection: undefined; + CameraAffineTransform: undefined; } // https://reactnavigation.org/docs/typescript#specifying-default-types-for-usenavigation-link-ref-etc diff --git a/src/functions/Core.ts b/src/functions/Core.ts index 8be5f5d..b4ff091 100644 --- a/src/functions/Core.ts +++ b/src/functions/Core.ts @@ -888,6 +888,15 @@ export type Core = { */ invoke(name: 'transform', src: Mat, dst: Mat, m: Mat): void; + /** + * Performs the affine transformation of an image + * @param name Function name. + * @param src input array that must have as many channels (1 to 4) as m.cols or m.cols-1. + * @param dst output array of the same size and depth as src; it has as many channels as m.rows + * @param m 2x3 floating-point matrix + */ + invoke(name: 'warpAffine', src: Mat, dst: Mat, m: Mat): void; + /** * Transposes a matrix * @param name Function name. From 1963dbce7f8bc7b046b5a1e1c88f6ff8551570c3 Mon Sep 17 00:00:00 2001 From: Thomas Coldwell <31568400+thomas-coldwell@users.noreply.github.com> Date: Fri, 27 Sep 2024 09:31:42 +0100 Subject: [PATCH 2/6] fix: remove unused --- example/src/examples/CameraAffineTransform.tsx | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/example/src/examples/CameraAffineTransform.tsx b/example/src/examples/CameraAffineTransform.tsx index 2ec9e9e..c196603 100644 --- a/example/src/examples/CameraAffineTransform.tsx +++ b/example/src/examples/CameraAffineTransform.tsx @@ -88,23 +88,6 @@ export function createSkiaImageFromData( ); } -// const multiply2x3Matrix = (a: number[][], b: number[][]) => { -// 'worklet'; -// const result = []; -// for (let i = 0; i < a.length; i++) { -// const row = []; -// for (let j = 0; j < b[0].length; j++) { -// let sum = 0; -// for (let k = 0; k < a[0].length; k++) { -// sum += a[i][k] * b[k][j]; -// } -// row.push(sum); -// } -// result.push(row); -// } -// return result; -// }; - export function CameraAffineTransform() { const device = useCameraDevice('back'); const { hasPermission, requestPermission } = useCameraPermission(); From 24e3f26b6bee4b7115b6fb6111d9df9dcfad2c45 Mon Sep 17 00:00:00 2001 From: Thomas Coldwell <31568400+thomas-coldwell@users.noreply.github.com> Date: Fri, 27 Sep 2024 09:50:07 +0100 Subject: [PATCH 3/6] fix: add `private: true` to support yarn installs see: https://github.com/yarnpkg/yarn/issues/8580 --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 6162b40..c89921b 100644 --- a/package.json +++ b/package.json @@ -96,6 +96,7 @@ "workspaces": [ "example" ], + "private": true, "packageManager": "yarn@4.4.1", "jest": { "preset": "react-native", From a3095641728eb9a82bd4329c5508e3131d6c1976 Mon Sep 17 00:00:00 2001 From: Thomas Coldwell <31568400+thomas-coldwell@users.noreply.github.com> Date: Fri, 27 Sep 2024 10:07:24 +0100 Subject: [PATCH 4/6] fix: add scale and offset to `convertTo` --- cpp/FOCV_Function.cpp | 12 ++++++++++-- src/functions/Core.ts | 11 ++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/cpp/FOCV_Function.cpp b/cpp/FOCV_Function.cpp index 98b213c..22ed6bc 100644 --- a/cpp/FOCV_Function.cpp +++ b/cpp/FOCV_Function.cpp @@ -1343,8 +1343,16 @@ jsi::Object FOCV_Function::invoke(jsi::Runtime& runtime, const jsi::Value* argum auto src = args.asMatPtr(1); auto dst = args.asMatPtr(2); auto rtype = args.asNumber(3); - - (*src).convertTo(*dst, rtype); + double scale = 1.0; + double offset = 0.0; + if (args.asNumber(4)) { + scale = args.asNumber(4); + } + if (args.asNumber(5)) { + offset = args.asNumber(5); + } + + (*src).convertTo(*dst, rtype, scale, offset); } break; } } catch (cv::Exception& e) { diff --git a/src/functions/Core.ts b/src/functions/Core.ts index b4ff091..cbe3cd9 100644 --- a/src/functions/Core.ts +++ b/src/functions/Core.ts @@ -919,6 +919,15 @@ export type Core = { * @param src input array. * @param dst output array of the same type as src * @param rtype desired output matrix type or, rather, the depth since the number of channels are the same as the input has; if rtype is negative, the output matrix will have the same type as the input. + * @param scale optional scale factor + * @param offset optional delta added to the scaled values */ - invoke(name: 'convertTo', src: Mat, dst: Mat, rtype: DataTypes): void; + invoke( + name: 'convertTo', + src: Mat, + dst: Mat, + rtype: DataTypes, + scale?: number, + offset?: number + ): void; }; From 9c51f12c5c90ae53d12cf94a5f07069d967cc7d4 Mon Sep 17 00:00:00 2001 From: Thomas Coldwell <31568400+thomas-coldwell@users.noreply.github.com> Date: Sat, 28 Sep 2024 15:51:26 +0100 Subject: [PATCH 5/6] fix: wip color and format conversion --- cpp/FOCV_Object.cpp | 44 +++++------ cpp/FOCV_Object.hpp | 2 +- cpp/react-native-fast-opencv.cpp | 45 +++++++---- .../src/examples/CameraAffineTransform.tsx | 78 +++++++++++++++---- example/src/examples/CameraPassthrough.tsx | 8 +- .../src/examples/CameraRealtimeDetection.tsx | 2 +- src/utils/UtilsFunctions.ts | 5 +- 7 files changed, 124 insertions(+), 60 deletions(-) diff --git a/cpp/FOCV_Object.cpp b/cpp/FOCV_Object.cpp index a348411..9d21326 100644 --- a/cpp/FOCV_Object.cpp +++ b/cpp/FOCV_Object.cpp @@ -29,7 +29,7 @@ constexpr uint64_t hashString(const char* str, size_t length) { return hash; } -jsi::Object FOCV_Object::create(jsi::Runtime& runtime, const jsi::Value* arguments) { +jsi::Object FOCV_Object::create(jsi::Runtime& runtime, const jsi::Value* arguments, size_t argCount) { std::string id = ""; std::string objectType = arguments[0].asString(runtime).utf8(runtime); @@ -38,29 +38,27 @@ jsi::Object FOCV_Object::create(jsi::Runtime& runtime, const jsi::Value* argumen int rows = arguments[1].asNumber(); int cols = arguments[2].asNumber(); int type = arguments[3].asNumber(); - - if(arguments[4].isObject()) { - auto rawArray = arguments[4].asObject(runtime); - auto array = rawArray.asArray(runtime); - - auto rawLength = rawArray.getProperty(runtime, "length"); - auto length = rawLength.asNumber(); - - std::vector vec; - - for(auto i = 0; i < length; i++) { - vec.push_back(array.getValueAtIndex(runtime, i).asNumber()); - } - - cv::Mat mat{vec, true}; - mat = mat.reshape(1, rows); - mat.convertTo(mat, type); - - id = FOCV_Storage::save(mat); - } else { - cv::Mat object(rows, cols, type); - id = FOCV_Storage::save(object); + + cv::Mat mat(rows, cols, type); + memset(mat.data, 0, rows * cols * mat.elemSize1()); + + if (argCount == 5 && arguments[4].isObject()) { + std::vector vec; + auto rawArray = arguments[4].asObject(runtime); + auto array = rawArray.asArray(runtime); + + auto rawLength = rawArray.getProperty(runtime, "length"); + auto length = rawLength.asNumber(); + + for(auto i = 0; i < length; i++) { + vec.push_back(array.getValueAtIndex(runtime, i).asNumber()); + } + memcpy(mat.data, vec.data(), vec.size() * mat.elemSize1()); + mat = mat.reshape(1, rows); + mat.convertTo(mat, type); } + + id = FOCV_Storage::save(mat); } break; case hashString("mat_vector", 10): { std::vector object; diff --git a/cpp/FOCV_Object.hpp b/cpp/FOCV_Object.hpp index 2e136ba..39a6ff6 100644 --- a/cpp/FOCV_Object.hpp +++ b/cpp/FOCV_Object.hpp @@ -30,7 +30,7 @@ using namespace facebook; class FOCV_Object { public: - static jsi::Object create(jsi::Runtime& runtime, const jsi::Value* arguments); + static jsi::Object create(jsi::Runtime& runtime, const jsi::Value* arguments, size_t argCount); static jsi::Object convertToJSI(jsi::Runtime& runtime, const jsi::Value* arguments); static jsi::Object copyObjectFromVector(jsi::Runtime& runtime, const jsi::Value* arguments); }; diff --git a/cpp/react-native-fast-opencv.cpp b/cpp/react-native-fast-opencv.cpp index 562d426..13fbf7a 100644 --- a/cpp/react-native-fast-opencv.cpp +++ b/cpp/react-native-fast-opencv.cpp @@ -47,36 +47,51 @@ jsi::Value OpenCVPlugin::get(jsi::Runtime& runtime, const jsi::PropNameID& propN double rows = arguments[0].asNumber(); double cols = arguments[1].asNumber(); int channels = arguments[2].asNumber(); - jsi::Object input = arguments[3].asObject(runtime); + std::string dataType = arguments[3].asString(runtime).utf8(runtime); + jsi::Object input = arguments[4].asObject(runtime); int type = -1; if (channels == 1) { - type = CV_8U; + type = dataType == "float32" ? CV_32F : CV_8U; } if (channels == 3) { - type = CV_8UC3; + type = dataType == "float32" ? CV_32FC3 : CV_8UC3; } if (channels == 4) { - type = CV_8UC4; + type = dataType == "float32" ? CV_32FC4 : CV_8UC4; } if (channels == -1) { throw std::runtime_error("Fast OpenCV Error: Invalid channel count passed to frameBufferToMat!"); } + + TypedArrayKind typedArrayType = dataType == "float32" ? TypedArrayKind::Float32Array : TypedArrayKind::Uint8Array; + + if (dataType == "float32") { + TypedArrayBase inputBuffer = getTypedArray(runtime, std::move(input)); + auto vec = inputBuffer.toVector(runtime); + + cv::Mat mat(rows, cols, type); + memcpy(mat.data, vec.data(), (int)rows * (int)cols * channels * 4); + auto id = FOCV_Storage::save(mat); + + return FOCV_JsiObject::wrap(runtime, "mat", id); + } + else { + TypedArrayBase inputBuffer = getTypedArray(runtime, std::move(input)); + auto vec = inputBuffer.toVector(runtime); - TypedArrayBase inputBuffer = getTypedArray(runtime, std::move(input)); - auto vec = inputBuffer.toVector(runtime); - - cv::Mat mat(rows, cols, type); - memcpy(mat.data, vec.data(), (int)rows * (int)cols * channels); - auto id = FOCV_Storage::save(mat); - - return FOCV_JsiObject::wrap(runtime, "mat", id); + cv::Mat mat(rows, cols, type); + memcpy(mat.data, vec.data(), (int)rows * (int)cols * channels); + auto id = FOCV_Storage::save(mat); + + return FOCV_JsiObject::wrap(runtime, "mat", id); + } }); } else if (propName == "base64ToMat") { return jsi::Function::createFromHostFunction( - runtime, jsi::PropNameID::forAscii(runtime, "frameBufferToMat"), 1, + runtime, jsi::PropNameID::forAscii(runtime, "base64ToMat"), 1, [=](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Object { @@ -123,8 +138,8 @@ jsi::Value OpenCVPlugin::get(jsi::Runtime& runtime, const jsi::PropNameID& propN runtime, jsi::PropNameID::forAscii(runtime, "createObject"), 1, [=](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Object { - - return FOCV_Object::create(runtime, arguments); + + return FOCV_Object::create(runtime, arguments, count); }); } else if (propName == "toJSValue") { diff --git a/example/src/examples/CameraAffineTransform.tsx b/example/src/examples/CameraAffineTransform.tsx index c196603..13de821 100644 --- a/example/src/examples/CameraAffineTransform.tsx +++ b/example/src/examples/CameraAffineTransform.tsx @@ -9,7 +9,12 @@ import { } from '@shopify/react-native-skia'; import { useEffect } from 'react'; import { SafeAreaView, StyleSheet, Text, View } from 'react-native'; -import { DataTypes, ObjectType, OpenCV } from 'react-native-fast-opencv'; +import { + ColorConversionCodes, + DataTypes, + ObjectType, + OpenCV, +} from 'react-native-fast-opencv'; import { useSharedValue } from 'react-native-reanimated'; import { Camera, @@ -22,10 +27,10 @@ import { useResizePlugin, type Options } from 'vision-camera-resize-plugin'; type PixelFormat = Options<'uint8'>['pixelFormat']; -const WIDTH = 300; -const HEIGHT = 300; -const TARGET_TYPE = 'uint8' as const; -const TARGET_FORMAT: PixelFormat = 'rgba'; +const WIDTH = 50; +const HEIGHT = 50; +const TARGET_TYPE = 'float32' as const; +const TARGET_FORMAT: PixelFormat = 'rgb'; let lastWarn: PixelFormat | undefined; lastWarn = undefined; @@ -114,6 +119,7 @@ export function CameraAffineTransform() { (frame) => { 'worklet'; + // Step 1. Resize the frame to the target size const resized = resize(frame, { scale: { width: WIDTH, @@ -130,18 +136,26 @@ export function CameraAffineTransform() { rotation: '90deg', }); - const frameMat = OpenCV.frameBufferToMat(HEIGHT, WIDTH, 4, resized); - const outputMat = OpenCV.createObject( - ObjectType.Mat, + // Step 2. Create an OpenCV Mat from the resized frame + const frameMat = OpenCV.frameBufferToMat( HEIGHT, WIDTH, - DataTypes.CV_8UC4 + 3, + 'float32', + resized ); + // Step 3. Apply affine transform to the frame + const transformedMat = OpenCV.createObject( + ObjectType.Mat, + HEIGHT, + WIDTH, + DataTypes.CV_32FC3, + undefined + ); const scaleX = frame.height / frame.width; const scaledFrameWidth = WIDTH * scaleX; const translateX = (scaledFrameWidth - WIDTH) / 2; - // 2x3 affine matrix has the following form (for scale and translation only): // | sx 0 tx | // | 0 sy ty | @@ -149,7 +163,6 @@ export function CameraAffineTransform() { [scaleX, 0.0, -translateX], [0.0, 1.0, 0.0], ]; - const transformMat = OpenCV.createObject( ObjectType.Mat, 2, @@ -157,14 +170,43 @@ export function CameraAffineTransform() { DataTypes.CV_32F, matrix.flat() ); + OpenCV.invoke('warpAffine', frameMat, transformedMat, transformMat); - OpenCV.invoke('warpAffine', frameMat, outputMat, transformMat); - - const output = OpenCV.matToBuffer(outputMat, 'uint8'); - const data = Skia.Data.fromBytes(output.buffer); + // Step 4. Convert the transformed frame to RGBA + const uint8TransformedMat = OpenCV.createObject( + ObjectType.Mat, + HEIGHT, + WIDTH, + DataTypes.CV_8UC3, + undefined + ); + OpenCV.invoke( + 'convertTo', + transformedMat, + uint8TransformedMat, + DataTypes.CV_8UC3, + 255.0, + 0 + ); + const rgbaOutputMat = OpenCV.createObject( + ObjectType.Mat, + HEIGHT, + WIDTH, + DataTypes.CV_8UC4, + undefined + ); + OpenCV.invoke( + 'cvtColor', + uint8TransformedMat, + rgbaOutputMat, + ColorConversionCodes.COLOR_RGB2RGBA + ); - updatePreviewImageFromData(data, TARGET_FORMAT).then(() => { - data.dispose(); + // Step 5. Convert the transformed frame to Skia Image + const output = OpenCV.matToBuffer(rgbaOutputMat, 'uint8'); + const outputData = Skia.Data.fromBytes(output.buffer); + updatePreviewImageFromData(outputData, 'rgba').then(() => { + outputData.dispose(); }); OpenCV.clearBuffers(); // REMEMBER TO CLEAN @@ -224,5 +266,7 @@ const styles = StyleSheet.create({ height: HEIGHT, borderColor: 'red', borderWidth: 2, + transformOrigin: 'bottom', + transform: [{ scale: 4 }], }, }); diff --git a/example/src/examples/CameraPassthrough.tsx b/example/src/examples/CameraPassthrough.tsx index 591e3de..b4fd74d 100644 --- a/example/src/examples/CameraPassthrough.tsx +++ b/example/src/examples/CameraPassthrough.tsx @@ -130,7 +130,13 @@ export function CameraPassthrough() { rotation: '90deg', }); - const frameMat = OpenCV.frameBufferToMat(HEIGHT, WIDTH, 4, resized); + const frameMat = OpenCV.frameBufferToMat( + HEIGHT, + WIDTH, + 4, + 'uint8', + resized + ); const output = OpenCV.matToBuffer(frameMat, 'uint8'); const data = Skia.Data.fromBytes(output.buffer); diff --git a/example/src/examples/CameraRealtimeDetection.tsx b/example/src/examples/CameraRealtimeDetection.tsx index cbdb078..17ec494 100644 --- a/example/src/examples/CameraRealtimeDetection.tsx +++ b/example/src/examples/CameraRealtimeDetection.tsx @@ -47,7 +47,7 @@ export function CameraRealtimeDetection() { dataType: 'uint8', }); - const src = OpenCV.frameBufferToMat(height, width, 3, resized); + const src = OpenCV.frameBufferToMat(height, width, 3, 'uint8', resized); const dst = OpenCV.createObject(ObjectType.Mat, 0, 0, DataTypes.CV_8U); const lowerBound = OpenCV.createObject(ObjectType.Scalar, 30, 60, 60); diff --git a/src/utils/UtilsFunctions.ts b/src/utils/UtilsFunctions.ts index d2fac63..9d220a1 100644 --- a/src/utils/UtilsFunctions.ts +++ b/src/utils/UtilsFunctions.ts @@ -12,11 +12,12 @@ export type UtilsFunctions = { * @param channels - the number of channels in the Mat * @param input - the byte array to convert */ - frameBufferToMat( + frameBufferToMat( rows: number, cols: number, channels: number, - input: Uint8Array + type: T, + input: T extends 'uint8' ? Uint8Array : Float32Array ): Mat; /** * Converts a base64 string to a Mat. From 08ac801573da045d206a15c9a52a26020fcf8713 Mon Sep 17 00:00:00 2001 From: Thomas Coldwell <31568400+thomas-coldwell@users.noreply.github.com> Date: Sat, 28 Sep 2024 15:55:32 +0100 Subject: [PATCH 6/6] fix: use index.tsx for package export --- package.json | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/package.json b/package.json index c89921b..ae72c7d 100644 --- a/package.json +++ b/package.json @@ -3,20 +3,8 @@ "version": "0.2.6", "description": "OpenCV port to React Native", "source": "./src/index.tsx", - "main": "./lib/commonjs/index.js", - "module": "./lib/module/index.js", - "exports": { - ".": { - "import": { - "types": "./lib/typescript/module/src/index.d.ts", - "default": "./lib/module/index.js" - }, - "require": { - "types": "./lib/typescript/commonjs/src/index.d.ts", - "default": "./lib/commonjs/index.js" - } - } - }, + "main": "./src/index.tsx", + "module": "./src/index.tsx", "files": [ "src", "lib", @@ -193,4 +181,4 @@ "languages": "cpp", "version": "0.40.0" } -} +} \ No newline at end of file