From c2524116083c576486b0df734d2e30a5e4d06f34 Mon Sep 17 00:00:00 2001 From: Ronald Hecker Date: Wed, 9 Apr 2025 16:27:38 +0200 Subject: [PATCH 01/12] Set openvino version to 25.1.0. Paths are probably not correct yet. Will adjust when release ov is released. --- .github/workflows/build-and-test.yml | 4 ++-- .github/workflows/build-linux-bindings.yml | 2 +- .github/workflows/build-macos-bindings.yml | 2 +- .github/workflows/build-windows-bindings.yml | 6 +++--- openvino_bindings/Dockerfile.ubuntu | 4 ++-- openvino_bindings/WORKSPACE | 2 +- openvino_bindings/third_party/model_api/model_api.BUILD | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 478d4007..215bd701 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -25,7 +25,7 @@ jobs: contents: write uses: ./.github/workflows/build-linux-bindings.yml with: - OPENVINO_VERSION: '2025.0' + OPENVINO_VERSION: '2025.1' build-windows-bindings: permissions: @@ -37,7 +37,7 @@ jobs: contents: write uses: ./.github/workflows/build-macos-bindings.yml with: - OPENVINO_VERSION: '2025.0' + OPENVINO_VERSION: '2025.1' package: name: Package combined release diff --git a/.github/workflows/build-linux-bindings.yml b/.github/workflows/build-linux-bindings.yml index a6d807a4..cab148fd 100644 --- a/.github/workflows/build-linux-bindings.yml +++ b/.github/workflows/build-linux-bindings.yml @@ -42,7 +42,7 @@ jobs: - name: Install OpenVINO run: | sudo mkdir -p /opt/intel - curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2025.0/linux/openvino_genai_ubuntu22_2025.0.0.0_x86_64.tar.gz \ + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2025.1/linux/openvino_genai_ubuntu22_2025.1.0.0_x86_64.tar.gz \ --output openvino.tgz && tar -xf openvino.tgz && \ sudo mv openvino_genai_ubuntu22_${{ inputs.OPENVINO_VERSION }}.0.0_x86_64 /opt/intel/openvino && rm openvino.tgz cd /opt/intel/openvino && sudo ./install_dependencies/install_openvino_dependencies.sh -y diff --git a/.github/workflows/build-macos-bindings.yml b/.github/workflows/build-macos-bindings.yml index 33ddcfc5..c2adc376 100644 --- a/.github/workflows/build-macos-bindings.yml +++ b/.github/workflows/build-macos-bindings.yml @@ -25,7 +25,7 @@ jobs: - name: Install OpenVINO run: | - curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2025.0/macos/openvino_genai_macos_12_6_2025.0.0.0_arm64.tar.gz \ + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2025.1/macos/openvino_genai_macos_12_6_2025.1.0.0_arm64.tar.gz \ -o /tmp/openvino.tar.gz sudo mkdir /opt/intel sudo tar -xvf /tmp/openvino.tar.gz -C /opt/intel diff --git a/.github/workflows/build-windows-bindings.yml b/.github/workflows/build-windows-bindings.yml index 548ee65a..a1672629 100644 --- a/.github/workflows/build-windows-bindings.yml +++ b/.github/workflows/build-windows-bindings.yml @@ -99,10 +99,10 @@ jobs: - name: Download and Install OpenVINO Runtime 24.6.0 shell: powershell run: | - Invoke-WebRequest -Uri https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2025.0/windows/openvino_genai_windows_2025.0.0.0_x86_64.zip -OutFile openvino_runtime.zip + Invoke-WebRequest -Uri https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2025.1/windows/openvino_genai_windows_2025.1.0.0_x86_64.zip -OutFile openvino_runtime.zip Expand-Archive -Path openvino_runtime.zip -DestinationPath C:/Intel/ - Rename-Item -Path "C:/Intel/openvino_genai_windows_2025.0.0.0_x86_64" -NewName "openvino_2025.0.0" - dir C:/Intel/openvino_2025.0.0/ + Rename-Item -Path "C:/Intel/openvino_genai_windows_2025.1.0.0_x86_64" -NewName "openvino_2025.1.0" + dir C:/Intel/openvino_2025.1.0/ # Step 11: Install Mediapipe Requirements - name: Install Mediapipe Requirements diff --git a/openvino_bindings/Dockerfile.ubuntu b/openvino_bindings/Dockerfile.ubuntu index da2ea0d6..79121e99 100644 --- a/openvino_bindings/Dockerfile.ubuntu +++ b/openvino_bindings/Dockerfile.ubuntu @@ -24,9 +24,9 @@ RUN curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_V rm -f /bazel/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh RUN cd /tmp && mkdir -p /opt/intel && \ - curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/pre-release/2025.0.0.0rc3/openvino_genai_ubuntu22_2025.0.0.0rc3_x86_64.tar.gz \ + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/pre-release/2025.1.0.0rc3/openvino_genai_ubuntu22_2025.1.0.0rc3_x86_64.tar.gz \ --output openvino.tgz && tar -xf openvino.tgz && \ - mv openvino_genai_ubuntu22_2025.0.0.0rc3_x86_64 /opt/intel/openvino && rm /tmp/openvino.tgz + mv openvino_genai_ubuntu22_2025.1.0.0rc3_x86_64 /opt/intel/openvino && rm /tmp/openvino.tgz RUN cd /opt/intel/openvino && ./install_dependencies/install_openvino_dependencies.sh -y diff --git a/openvino_bindings/WORKSPACE b/openvino_bindings/WORKSPACE index db8d9f10..ce4da2f7 100644 --- a/openvino_bindings/WORKSPACE +++ b/openvino_bindings/WORKSPACE @@ -35,7 +35,7 @@ new_local_repository( new_local_repository( name = "openvino_windows", build_file = "//third_party/openvino:windows.BUILD", - path = "C:/Intel/openvino_2025.0.0/runtime", + path = "C:/Intel/openvino_2025.1.0/runtime", ) new_local_repository( diff --git a/openvino_bindings/third_party/model_api/model_api.BUILD b/openvino_bindings/third_party/model_api/model_api.BUILD index b58cd1d1..bce19868 100644 --- a/openvino_bindings/third_party/model_api/model_api.BUILD +++ b/openvino_bindings/third_party/model_api/model_api.BUILD @@ -61,7 +61,7 @@ cmake( ], cache_entries = { "CMAKE_POSITION_INDEPENDENT_CODE": "ON", - "OpenVINO_DIR": "C:/Intel/openvino_2025.0.0/runtime/cmake", + "OpenVINO_DIR": "C:/Intel/openvino_2025.1.0/runtime/cmake", "OpenCV_DIR": "C:/opencv/build", }, lib_source = ":all_srcs", From ddec0d8c6b5bd0ccd255006113a0ffcbaf562640 Mon Sep 17 00:00:00 2001 From: Ronald Hecker Date: Tue, 15 Apr 2025 06:30:17 +0200 Subject: [PATCH 02/12] Remove reference to openvino 2025.1 RC3 from docker --- openvino_bindings/Dockerfile.ubuntu | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openvino_bindings/Dockerfile.ubuntu b/openvino_bindings/Dockerfile.ubuntu index 79121e99..9a4f8d48 100644 --- a/openvino_bindings/Dockerfile.ubuntu +++ b/openvino_bindings/Dockerfile.ubuntu @@ -24,9 +24,9 @@ RUN curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_V rm -f /bazel/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh RUN cd /tmp && mkdir -p /opt/intel && \ - curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/pre-release/2025.1.0.0rc3/openvino_genai_ubuntu22_2025.1.0.0rc3_x86_64.tar.gz \ + curl -L https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2025.1/linux/openvino_genai_ubuntu22_2025.1.0.0_x86_64.tar.gz \ --output openvino.tgz && tar -xf openvino.tgz && \ - mv openvino_genai_ubuntu22_2025.1.0.0rc3_x86_64 /opt/intel/openvino && rm /tmp/openvino.tgz + mv openvino_genai_ubuntu22_2025.1.0.0_x86_64 /opt/intel/openvino && rm /tmp/openvino.tgz RUN cd /opt/intel/openvino && ./install_dependencies/install_openvino_dependencies.sh -y From 477a79064c9b7086cbeaf212bb3fccf99636a1f6 Mon Sep 17 00:00:00 2001 From: Ronald Hecker Date: Fri, 25 Apr 2025 17:08:43 +0200 Subject: [PATCH 03/12] Minor changes Allow npu for manually imported models Change name of devices --- lib/importers/model_directory_importer.dart | 5 ++++- lib/importers/model_manifest.dart | 2 +- openvino_bindings/src/bindings.cc | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/importers/model_directory_importer.dart b/lib/importers/model_directory_importer.dart index 5218e4c4..f6e629e2 100644 --- a/lib/importers/model_directory_importer.dart +++ b/lib/importers/model_directory_importer.dart @@ -32,7 +32,10 @@ class ModelDirImporter extends Importer { final source = File(projectJson).readAsStringSync(); final migrated = migrationManager.migrate(jsonDecode(source)); - project = Project.fromJson(migrated, directory); + project = Project.fromJson(migrated, directory); + if (project is PublicProject) { // Always allow npu when importing a model manually + (project as PublicProject).manifest.npuEnabled = true; + } } return project!; } diff --git a/lib/importers/model_manifest.dart b/lib/importers/model_manifest.dart index e60ff678..f5794018 100644 --- a/lib/importers/model_manifest.dart +++ b/lib/importers/model_manifest.dart @@ -16,7 +16,7 @@ class ModelManifest { final String description; final String task; final String author; - final bool npuEnabled; + bool npuEnabled; String? architecture; ModelManifest({ diff --git a/openvino_bindings/src/bindings.cc b/openvino_bindings/src/bindings.cc index 6d8329bd..72f56f63 100644 --- a/openvino_bindings/src/bindings.cc +++ b/openvino_bindings/src/bindings.cc @@ -465,7 +465,7 @@ StatusOrDevices* getAvailableDevices() { devices[0] = {"AUTO", "auto"}; for (int i = 0; i < device_ids.size(); i++) { auto device_name = core.get_property(device_ids[i], ov::device::full_name); - devices[i + 1] = { strdup(device_ids[i].c_str()), strdup(device_name.c_str()) }; + devices[i + 1] = { strdup(device_ids[i].c_str()), strdup((device_ids[i] + "(" + device_name + ")").c_str()) }; } return new StatusOrDevices{OkStatus, "", devices, (int)device_ids.size() + 1}; From 405c241d6c0366a985a1794d8684fbe57b8bfd8a Mon Sep 17 00:00:00 2001 From: Ronald Hecker Date: Mon, 28 Apr 2025 11:04:25 +0200 Subject: [PATCH 04/12] Implement camera view using the canvas --- lib/annotation.dart | 5 +- lib/interop/device.dart | 4 +- lib/interop/generated_bindings.dart | 164 +++++++++++++----- lib/interop/graph_runner.dart | 22 +-- lib/interop/llm_inference.dart | 16 +- lib/interop/openvino_bindings.dart | 3 +- lib/interop/sentence_transformer.dart | 6 +- lib/interop/speech_to_text.dart | 8 +- lib/interop/tti_inference.dart | 8 +- lib/interop/utils.dart | 2 +- lib/interop/vlm_inference.dart | 14 +- .../computer_vision/widgets/camera_view.dart | 27 +-- lib/utils/image_graph_builder.dart | 1 + lib/widgets/canvas/canvas_painter.dart | 2 +- openvino_bindings/src/bindings.cc | 8 +- openvino_bindings/src/bindings.h | 4 +- .../serialization_calculators.cc | 6 + .../serialization/serialization_calculators.h | 1 + 18 files changed, 197 insertions(+), 104 deletions(-) diff --git a/lib/annotation.dart b/lib/annotation.dart index ce03f492..eab325a5 100644 --- a/lib/annotation.dart +++ b/lib/annotation.dart @@ -177,11 +177,12 @@ class ImageInferenceResult { final String? csv; final Map? json; final String? overlay; + final String? source; - ImageInferenceResult({this.csv, this.json, this.overlay}); + ImageInferenceResult({this.csv, this.json, this.overlay, this.source}); factory ImageInferenceResult.fromJson(Map output) { - return ImageInferenceResult(csv: output["csv"], json: output["json"], overlay: output["overlay"]); + return ImageInferenceResult(csv: output["csv"], json: output["json"], overlay: output["overlay"], source: output["source"]); } List parseAnnotations() { diff --git a/lib/interop/device.dart b/lib/interop/device.dart index c9a68276..5b25bf3d 100644 --- a/lib/interop/device.dart +++ b/lib/interop/device.dart @@ -19,7 +19,7 @@ class Device { final result = await Isolate.run(() { final status = deviceOV.getAvailableDevices(); - if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) { + if (status.ref.status != StatusEnum.OkStatus) { throw "GetAvailableDevices error: ${status.ref.status} ${status.ref.message.toDartString()}"; } @@ -48,7 +48,7 @@ class CameraDevice { final result = await Isolate.run(() { final status = deviceOV.getAvailableCameraDevices(); - if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) { + if (status.ref.status != StatusEnum.OkStatus) { throw "GetAvailableDevices error: ${status.ref.status} ${status.ref.message.toDartString()}"; } diff --git a/lib/interop/generated_bindings.dart b/lib/interop/generated_bindings.dart index 732ce711..7127cbec 100644 --- a/lib/interop/generated_bindings.dart +++ b/lib/interop/generated_bindings.dart @@ -350,7 +350,7 @@ class OpenVINO { .asFunction Function(CTTIInference)>(); ffi.Pointer ttiInferenceClose( - CTTIInference instance, + CLLMInference instance, ) { return _ttiInferenceClose( instance, @@ -358,10 +358,10 @@ class OpenVINO { } late final _ttiInferenceClosePtr = - _lookup Function(CTTIInference)>>( + _lookup Function(CLLMInference)>>( 'ttiInferenceClose'); late final _ttiInferenceClose = _ttiInferenceClosePtr - .asFunction Function(CTTIInference)>(); + .asFunction Function(CLLMInference)>(); ffi.Pointer vlmInferenceOpen( ffi.Pointer model_path, @@ -535,6 +535,7 @@ class OpenVINO { bool json, bool csv, bool overlay, + bool source, ) { return _graphRunnerQueueSerializationOutput( instance, @@ -543,6 +544,7 @@ class OpenVINO { json, csv, overlay, + source, ); } @@ -554,11 +556,12 @@ class OpenVINO { ffi.Int, ffi.Bool, ffi.Bool, + ffi.Bool, ffi.Bool)>>('graphRunnerQueueSerializationOutput'); late final _graphRunnerQueueSerializationOutput = _graphRunnerQueueSerializationOutputPtr.asFunction< ffi.Pointer Function(CGraphRunner, ffi.Pointer, - int, bool, bool, bool)>(); + int, bool, bool, bool, bool)>(); ffi.Pointer graphRunnerGet( CGraphRunner instance, @@ -596,6 +599,7 @@ class OpenVINO { bool json, bool csv, bool overlay, + bool source, ) { return _graphRunnerStartCamera( instance, @@ -604,6 +608,7 @@ class OpenVINO { json, csv, overlay, + source, ); } @@ -615,10 +620,11 @@ class OpenVINO { ImageInferenceCallbackFunction, ffi.Bool, ffi.Bool, + ffi.Bool, ffi.Bool)>>('graphRunnerStartCamera'); late final _graphRunnerStartCamera = _graphRunnerStartCameraPtr.asFunction< ffi.Pointer Function(CGraphRunner, int, - ImageInferenceCallbackFunction, bool, bool, bool)>(); + ImageInferenceCallbackFunction, bool, bool, bool, bool)>(); ffi.Pointer graphRunnerGetTimestamp( CGraphRunner instance, @@ -824,6 +830,48 @@ class OpenVINO { _handle_exceptionsPtr.asFunction Function()>(); } +typedef va_list = ffi.Pointer; +typedef ptrdiff_t = ffi.LongLong; +typedef Dartptrdiff_t = int; +typedef errno_t = ffi.Int; +typedef Darterrno_t = int; +typedef wint_t = ffi.UnsignedShort; +typedef Dartwint_t = int; +typedef wctype_t = ffi.UnsignedShort; +typedef Dartwctype_t = int; +typedef __time32_t = ffi.Long; +typedef Dart__time32_t = int; +typedef __time64_t = ffi.LongLong; +typedef Dart__time64_t = int; + +final class __crt_locale_data extends ffi.Opaque {} + +final class __crt_multibyte_data extends ffi.Opaque {} + +final class __crt_locale_pointers extends ffi.Struct { + external ffi.Pointer<__crt_locale_data> locinfo; + + external ffi.Pointer<__crt_multibyte_data> mbcinfo; +} + +typedef _locale_t = ffi.Pointer<__crt_locale_pointers>; + +final class _Mbstatet extends ffi.Struct { + @ffi.UnsignedLong() + external int _Wchar; + + @ffi.UnsignedShort() + external int _Byte; + + @ffi.UnsignedShort() + external int _State; +} + +typedef mbstate_t = _Mbstatet; +typedef time_t = __time64_t; +typedef rsize_t = ffi.Size; +typedef Dartrsize_t = int; + enum StatusEnum { OkStatus(0), ErrorStatus(-1), @@ -952,6 +1000,14 @@ final class VLMStringWithMetrics extends ffi.Struct { external VLMMetrics metrics; } +typedef CImageInference = ffi.Pointer; +typedef CGraphRunner = ffi.Pointer; +typedef CSpeechToText = ffi.Pointer; +typedef CLLMInference = ffi.Pointer; +typedef CTTIInference = ffi.Pointer; +typedef CSentenceTransformer = ffi.Pointer; +typedef CVLMInference = ffi.Pointer; + final class Device extends ffi.Struct { external ffi.Pointer id; @@ -977,14 +1033,18 @@ final class TranscriptionChunk extends ffi.Struct { final class Status extends ffi.Struct { @ffi.Int() - external int status; + external int statusAsInt; + + StatusEnum get status => StatusEnum.fromValue(statusAsInt); external ffi.Pointer message; } final class StatusOrString extends ffi.Struct { @ffi.Int() - external int status; + external int statusAsInt; + + StatusEnum get status => StatusEnum.fromValue(statusAsInt); external ffi.Pointer message; @@ -993,7 +1053,9 @@ final class StatusOrString extends ffi.Struct { final class StatusOrBool extends ffi.Struct { @ffi.Int() - external int status; + external int statusAsInt; + + StatusEnum get status => StatusEnum.fromValue(statusAsInt); external ffi.Pointer message; @@ -1003,7 +1065,9 @@ final class StatusOrBool extends ffi.Struct { final class StatusOrInt extends ffi.Struct { @ffi.Int() - external int status; + external int statusAsInt; + + StatusEnum get status => StatusEnum.fromValue(statusAsInt); external ffi.Pointer message; @@ -1013,51 +1077,53 @@ final class StatusOrInt extends ffi.Struct { final class StatusOrGraphRunner extends ffi.Struct { @ffi.Int() - external int status; + external int statusAsInt; + + StatusEnum get status => StatusEnum.fromValue(statusAsInt); external ffi.Pointer message; external CGraphRunner value; } -typedef CGraphRunner = ffi.Pointer; - final class StatusOrSentenceTransformer extends ffi.Struct { @ffi.Int() - external int status; + external int statusAsInt; + + StatusEnum get status => StatusEnum.fromValue(statusAsInt); external ffi.Pointer message; external CSentenceTransformer value; } -typedef CSentenceTransformer = ffi.Pointer; - final class StatusOrSpeechToText extends ffi.Struct { @ffi.Int() - external int status; + external int statusAsInt; + + StatusEnum get status => StatusEnum.fromValue(statusAsInt); external ffi.Pointer message; external CSpeechToText value; } -typedef CSpeechToText = ffi.Pointer; - final class StatusOrLLMInference extends ffi.Struct { @ffi.Int() - external int status; + external int statusAsInt; + + StatusEnum get status => StatusEnum.fromValue(statusAsInt); external ffi.Pointer message; external CLLMInference value; } -typedef CLLMInference = ffi.Pointer; - final class StatusOrTTIInference extends ffi.Struct { @ffi.Int() - external int status; + external int statusAsInt; + + StatusEnum get status => StatusEnum.fromValue(statusAsInt); external ffi.Pointer message; @@ -1066,7 +1132,9 @@ final class StatusOrTTIInference extends ffi.Struct { final class StatusOrVLMInference extends ffi.Struct { @ffi.Int() - external int status; + external int statusAsInt; + + StatusEnum get status => StatusEnum.fromValue(statusAsInt); external ffi.Pointer message; @@ -1075,7 +1143,9 @@ final class StatusOrVLMInference extends ffi.Struct { final class StatusOrModelResponse extends ffi.Struct { @ffi.Int() - external int status; + external int statusAsInt; + + StatusEnum get status => StatusEnum.fromValue(statusAsInt); external ffi.Pointer message; @@ -1086,7 +1156,9 @@ final class StatusOrModelResponse extends ffi.Struct { final class StatusOrWhisperModelResponse extends ffi.Struct { @ffi.Int() - external int status; + external int statusAsInt; + + StatusEnum get status => StatusEnum.fromValue(statusAsInt); external ffi.Pointer message; @@ -1102,7 +1174,9 @@ final class StatusOrWhisperModelResponse extends ffi.Struct { final class StatusOrTTIModelResponse extends ffi.Struct { @ffi.Int() - external int status; + external int statusAsInt; + + StatusEnum get status => StatusEnum.fromValue(statusAsInt); external ffi.Pointer message; @@ -1113,7 +1187,9 @@ final class StatusOrTTIModelResponse extends ffi.Struct { final class StatusOrEmbeddings extends ffi.Struct { @ffi.Int() - external int status; + external int statusAsInt; + + StatusEnum get status => StatusEnum.fromValue(statusAsInt); external ffi.Pointer message; @@ -1125,7 +1201,9 @@ final class StatusOrEmbeddings extends ffi.Struct { final class StatusOrVLMModelResponse extends ffi.Struct { @ffi.Int() - external int status; + external int statusAsInt; + + StatusEnum get status => StatusEnum.fromValue(statusAsInt); external ffi.Pointer message; @@ -1136,7 +1214,9 @@ final class StatusOrVLMModelResponse extends ffi.Struct { final class StatusOrDevices extends ffi.Struct { @ffi.Int() - external int status; + external int statusAsInt; + + StatusEnum get status => StatusEnum.fromValue(statusAsInt); external ffi.Pointer message; @@ -1148,7 +1228,9 @@ final class StatusOrDevices extends ffi.Struct { final class StatusOrCameraDevices extends ffi.Struct { @ffi.Int() - external int status; + external int statusAsInt; + + StatusEnum get status => StatusEnum.fromValue(statusAsInt); external ffi.Pointer message; @@ -1158,23 +1240,21 @@ final class StatusOrCameraDevices extends ffi.Struct { external int size; } -typedef LLMInferenceCallbackFunction - = ffi.Pointer>; +typedef ImageInferenceCallbackFunctionFunction = ffi.Void Function( + ffi.Pointer); +typedef DartImageInferenceCallbackFunctionFunction = void Function( + ffi.Pointer); +typedef ImageInferenceCallbackFunction + = ffi.Pointer>; typedef LLMInferenceCallbackFunctionFunction = ffi.Void Function( ffi.Pointer); typedef DartLLMInferenceCallbackFunctionFunction = void Function( ffi.Pointer); -typedef CTTIInference = ffi.Pointer; -typedef CVLMInference = ffi.Pointer; -typedef VLMInferenceCallbackFunction - = ffi.Pointer>; +typedef LLMInferenceCallbackFunction + = ffi.Pointer>; typedef VLMInferenceCallbackFunctionFunction = ffi.Void Function( ffi.Pointer); typedef DartVLMInferenceCallbackFunctionFunction = void Function( ffi.Pointer); -typedef ImageInferenceCallbackFunction - = ffi.Pointer>; -typedef ImageInferenceCallbackFunctionFunction = ffi.Void Function( - ffi.Pointer); -typedef DartImageInferenceCallbackFunctionFunction = void Function( - ffi.Pointer); +typedef VLMInferenceCallbackFunction + = ffi.Pointer>; diff --git a/lib/interop/graph_runner.dart b/lib/interop/graph_runner.dart index 7f24d795..b7133e20 100644 --- a/lib/interop/graph_runner.dart +++ b/lib/interop/graph_runner.dart @@ -28,7 +28,7 @@ class GraphRunner { return status; }); - if (StatusEnum.fromValue(result.ref.status) != StatusEnum.OkStatus) { + if (result.ref.status != StatusEnum.OkStatus) { throw "GraphRunner::Init error: ${result.ref.status} ${result.ref.message.toDartString()}"; } @@ -38,7 +38,7 @@ class GraphRunner { int getTimestamp() { final status = ov.graphRunnerGetTimestamp(instance.ref.value); - if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) { + if (status.ref.status != StatusEnum.OkStatus) { throw "GraphRunner::get error: ${status.ref.status} ${status.ref.message.toDartString()}"; } final content = status.ref.value; @@ -51,7 +51,7 @@ class GraphRunner { return await Isolate.run(() { final status = ov.graphRunnerGet(Pointer.fromAddress(instanceAddress)); - if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) { + if (status.ref.status != StatusEnum.OkStatus) { throw "GraphRunner::get error: ${status.ref.status} ${status.ref.message.toDartString()}"; } final content = status.ref.value.toDartString(); @@ -62,7 +62,7 @@ class GraphRunner { Future startCamera(int deviceIndex, Function(String) callback, SerializationOutput output) async { void wrapCallback(Pointer ptr) { - if (StatusEnum.fromValue(ptr.ref.status) != StatusEnum.OkStatus) { + if (ptr.ref.status != StatusEnum.OkStatus) { // TODO(RHeckerIntel): instead of throw, call an onError callback. throw "ImageInference infer error: ${ptr.ref.status} ${ptr.ref.message.toDartString()}"; } @@ -73,8 +73,8 @@ class GraphRunner { nativeListener?.close(); nativeListener = NativeCallable.listener(wrapCallback); final nativeFunction = nativeListener!.nativeFunction; - final status = ov.graphRunnerStartCamera(instance.ref.value, deviceIndex, nativeFunction, output.json, output.csv, output.overlay); - if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) { + final status = ov.graphRunnerStartCamera(instance.ref.value, deviceIndex, nativeFunction, output.json, output.csv, output.overlay, output.source); + if (status.ref.status != StatusEnum.OkStatus) { throw "GraphRunner::StartCamera error: ${status.ref.status} ${status.ref.message.toDartString()}"; } } @@ -82,7 +82,7 @@ class GraphRunner { int instanceAddress = instance.ref.value.address; await Isolate.run(() { final status = ov.graphRunnerStopCamera(Pointer.fromAddress(instanceAddress)); - switch(StatusEnum.fromValue(status.ref.status)) { + switch(status.ref.status) { case StatusEnum.OkStatus: case StatusEnum.ErrorStatus: //Fail gracefully since race condition could happen with stopping the camera and we dont care about that break; @@ -103,7 +103,7 @@ class GraphRunner { final status = ov.graphRunnerQueueImage(Pointer.fromAddress(instanceAddress), nodeNamePtr, timestamp, _data, file.lengthInBytes); calloc.free(nodeNamePtr); - if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) { + if (status.ref.status != StatusEnum.OkStatus) { throw "QueueImage error: ${status.ref.status} ${status.ref.message.toDartString()}"; } }); @@ -113,10 +113,10 @@ class GraphRunner { int instanceAddress = instance.ref.value.address; await Isolate.run(() { final nodeNamePtr = nodeName.toNativeUtf8(); - final status = ov.graphRunnerQueueSerializationOutput(Pointer.fromAddress(instanceAddress), nodeNamePtr, timestamp, output.json, output.csv, output.overlay); + final status = ov.graphRunnerQueueSerializationOutput(Pointer.fromAddress(instanceAddress), nodeNamePtr, timestamp, output.json, output.csv, output.overlay, output.source); calloc.free(nodeNamePtr); - if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) { + if (status.ref.status != StatusEnum.OkStatus) { throw "QueueSerializationOutput error: ${status.ref.status} ${status.ref.message.toDartString()}"; } }); @@ -126,7 +126,7 @@ class GraphRunner { int instanceAddress = instance.ref.value.address; await Isolate.run(() { final status = ov.graphRunnerStop(Pointer.fromAddress(instanceAddress)); - if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) { + if (status.ref.status != StatusEnum.OkStatus) { throw "GraphRunner::stop error: ${status.ref.status} ${status.ref.message.toDartString()}"; } }); diff --git a/lib/interop/llm_inference.dart b/lib/interop/llm_inference.dart index 2a40f777..7c44fd10 100644 --- a/lib/interop/llm_inference.dart +++ b/lib/interop/llm_inference.dart @@ -28,7 +28,7 @@ class LLMInference { }); print("${result.ref.status}, ${result.ref.message}"); - if (StatusEnum.fromValue(result.ref.status) != StatusEnum.OkStatus) { + if (result.ref.status != StatusEnum.OkStatus) { throw "LLMInference open error: ${result.ref.status} ${result.ref.message.toDartString()}"; } @@ -46,7 +46,7 @@ class LLMInference { }) ; - if (StatusEnum.fromValue(result.ref.status) != StatusEnum.OkStatus) { + if (result.ref.status != StatusEnum.OkStatus) { throw "LLMInference prompt error: ${result.ref.status} ${result.ref.message.toDartString()}"; } @@ -56,7 +56,7 @@ class LLMInference { Future setListener(void Function(String) callback) async{ int instanceAddress = instance.ref.value.address; void localCallback(Pointer ptr) { - if (StatusEnum.fromValue(ptr.ref.status) != StatusEnum.OkStatus) { + if (ptr.ref.status != StatusEnum.OkStatus) { // TODO(RHeckerIntel): instead of throw, call an onError callback. throw "LLM Callback error: ${ptr.ref.status} ${ptr.ref.message.toDartString()}"; } @@ -66,7 +66,7 @@ class LLMInference { nativeListener?.close(); nativeListener = NativeCallable.listener(localCallback); final status = llmOV.llmInferenceSetListener(Pointer.fromAddress(instanceAddress), nativeListener!.nativeFunction); - if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) { + if (status.ref.status != StatusEnum.OkStatus) { // TODO(RHeckerIntel): instead of throw, call an onError callback. throw "LLM setListener error: ${status.ref.status} ${status.ref.message.toDartString()}"; } @@ -76,7 +76,7 @@ class LLMInference { void forceStop() { final status = llmOV.llmInferenceForceStop(instance.ref.value); - if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) { + if (status.ref.status != StatusEnum.OkStatus) { throw "LLM Force Stop error: ${status.ref.status} ${status.ref.message.toDartString()}"; } } @@ -84,7 +84,7 @@ class LLMInference { String getTokenizerConfig() { final status = llmOV.llmInferenceGetTokenizerConfig(instance.ref.value); - if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) { + if (status.ref.status != StatusEnum.OkStatus) { throw "LLM get Chat template error: ${status.ref.status} ${status.ref.message.toDartString()}"; } @@ -98,7 +98,7 @@ class LLMInference { final status = llmOV.llmInferenceClose(instance.ref.value); nativeListener?.close(); - if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) { + if (status.ref.status != StatusEnum.OkStatus) { throw "Close error: ${status.ref.status} ${status.ref.message.toDartString()}"; } llmOV.freeStatus(status); @@ -107,7 +107,7 @@ class LLMInference { void clearHistory() { final status = llmOV.llmInferenceClearHistory(instance.ref.value); - if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) { + if (status.ref.status != StatusEnum.OkStatus) { throw "Clear History: ${status.ref.status} ${status.ref.message.toDartString()}"; } } diff --git a/lib/interop/openvino_bindings.dart b/lib/interop/openvino_bindings.dart index d9733135..00802bb4 100644 --- a/lib/interop/openvino_bindings.dart +++ b/lib/interop/openvino_bindings.dart @@ -14,8 +14,9 @@ class SerializationOutput { bool csv; bool json; bool overlay; + bool source; - SerializationOutput({this.csv = false, this.json = false, this.overlay = false}); + SerializationOutput({this.csv = false, this.json = false, this.overlay = false, this.source = false}); bool any() => csv || json || overlay; diff --git a/lib/interop/sentence_transformer.dart b/lib/interop/sentence_transformer.dart index b125f2d1..f5c3e357 100644 --- a/lib/interop/sentence_transformer.dart +++ b/lib/interop/sentence_transformer.dart @@ -27,7 +27,7 @@ class SentenceTransformer { }); - if (StatusEnum.fromValue(result.ref.status) != StatusEnum.OkStatus) { + if (result.ref.status != StatusEnum.OkStatus) { throw "SentenceTransformer open error: ${result.ref.status} ${result.ref.message.toDartString()}"; } return SentenceTransformer(result); @@ -43,7 +43,7 @@ class SentenceTransformer { return status; }); - if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) { + if (status.ref.status != StatusEnum.OkStatus) { throw "SentenceTransformer generate error: ${status.ref.status} ${status.ref.message.toDartString()}"; } @@ -59,7 +59,7 @@ class SentenceTransformer { void close() { final status = ov.sentenceTransformerClose(instance.ref.value); - if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) { + if (status.ref.status != StatusEnum.OkStatus) { throw "Close error: ${status.ref.status} ${status.ref.message.toDartString()}"; } ov.freeStatus(status); diff --git a/lib/interop/speech_to_text.dart b/lib/interop/speech_to_text.dart index d2607e4b..3db1b678 100644 --- a/lib/interop/speech_to_text.dart +++ b/lib/interop/speech_to_text.dart @@ -27,7 +27,7 @@ class SpeechToText { }); print("${result.ref.status}, ${result.ref.message}"); - if (StatusEnum.fromValue(result.ref.status) != StatusEnum.OkStatus) { + if (result.ref.status != StatusEnum.OkStatus) { throw "SpeechToText open error: ${result.ref.status} ${result.ref.message.toDartString()}"; } @@ -44,7 +44,7 @@ class SpeechToText { return status; }); - if (StatusEnum.fromValue(result.ref.status) != StatusEnum.OkStatus) { + if (result.ref.status != StatusEnum.OkStatus) { throw "SpeechToText LoadVideo error: ${result.ref.status} ${result.ref.message.toDartString()}"; } } @@ -54,7 +54,7 @@ class SpeechToText { final status = ov.speechToTextVideoDuration(Pointer.fromAddress(instanceAddress)); return status; }); - if (StatusEnum.fromValue(result.ref.status) != StatusEnum.OkStatus) { + if (result.ref.status != StatusEnum.OkStatus) { throw "SpeechToText VideoDuration error: ${result.ref.status} ${result.ref.message.toDartString()}"; } return result.ref.value; @@ -70,7 +70,7 @@ class SpeechToText { return status; }); - if (StatusEnum.fromValue(result.ref.status) != StatusEnum.OkStatus) { + if (result.ref.status != StatusEnum.OkStatus) { throw "SpeechToText LoadVideo error: ${result.ref.status} ${result.ref.message.toDartString()}"; } diff --git a/lib/interop/tti_inference.dart b/lib/interop/tti_inference.dart index 527fcd1e..1ae6976b 100644 --- a/lib/interop/tti_inference.dart +++ b/lib/interop/tti_inference.dart @@ -30,7 +30,7 @@ class TTIInference { }); print("${result.ref.status}, ${result.ref.message}"); - if (StatusEnum.fromValue(result.ref.status) != StatusEnum.OkStatus) { + if (result.ref.status != StatusEnum.OkStatus) { throw "TTIInference open error: ${result.ref.status} ${result.ref.message.toDartString()}"; } @@ -52,7 +52,7 @@ class TTIInference { return status; }); - if (StatusEnum.fromValue(result.ref.status) != StatusEnum.OkStatus) { + if (result.ref.status != StatusEnum.OkStatus) { throw "TTIInference prompt error: ${result.ref.status} ${result.ref.message.toDartString()}"; } @@ -63,7 +63,7 @@ class TTIInference { bool hasModelIndex() { final status = ttiOV.ttiInferenceHasModelIndex(instance.ref.value); - if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) { + if (status.ref.status != StatusEnum.OkStatus) { throw "TTI Chat template error: ${status.ref.status} ${status.ref.message.toDartString()}"; } @@ -73,7 +73,7 @@ class TTIInference { void close() { final status = ttiOV.ttiInferenceClose(instance.ref.value); - if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) { + if (status.ref.status != StatusEnum.OkStatus) { throw "Close error: ${status.ref.status} ${status.ref.message.toDartString()}"; } ttiOV.freeStatus(status); diff --git a/lib/interop/utils.dart b/lib/interop/utils.dart index 1ec08064..1553a4ec 100644 --- a/lib/interop/utils.dart +++ b/lib/interop/utils.dart @@ -19,7 +19,7 @@ class InteropUtils { calloc.free(modelPathPtr); calloc.free(outputPathPtr); - if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) { + if (status.ref.status != StatusEnum.OkStatus) { throw "Serialize error: ${status.ref.status} ${status.ref.message.toDartString()}"; } print("Serialization done"); diff --git a/lib/interop/vlm_inference.dart b/lib/interop/vlm_inference.dart index 7a72e194..4ae29759 100644 --- a/lib/interop/vlm_inference.dart +++ b/lib/interop/vlm_inference.dart @@ -31,7 +31,7 @@ class VLMInference { }); print("${result.ref.status}, ${result.ref.message}"); - if (StatusEnum.fromValue(result.ref.status) != StatusEnum.OkStatus) { + if (result.ref.status != StatusEnum.OkStatus) { throw "VLMInference open error: ${result.ref.status} ${result.ref.message.toDartString()}"; } @@ -41,7 +41,7 @@ class VLMInference { Future setListener(void Function(String) callback) async{ int instanceAddress = instance.ref.value.address; void localCallback(Pointer ptr) { - if (StatusEnum.fromValue(ptr.ref.status) != StatusEnum.OkStatus) { + if (ptr.ref.status != StatusEnum.OkStatus) { // TODO(RHeckerIntel): instead of throw, call an onError callback. throw "VLM Callback error: ${ptr.ref.status} ${ptr.ref.message.toDartString()}"; } @@ -51,7 +51,7 @@ class VLMInference { nativeListener?.close(); nativeListener = NativeCallable.listener(localCallback); final status = vlmOV.vlmInferenceSetListener(Pointer.fromAddress(instanceAddress), nativeListener!.nativeFunction); - if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) { + if (status.ref.status != StatusEnum.OkStatus) { // TODO(RHeckerIntel): instead of throw, call an onError callback. throw "VLM setListener error: ${status.ref.status} ${status.ref.message.toDartString()}"; } @@ -72,7 +72,7 @@ class VLMInference { return status; }); - if (StatusEnum.fromValue(result.ref.status) != StatusEnum.OkStatus) { + if (result.ref.status != StatusEnum.OkStatus) { var msg = result.ref.message; var status = result.ref.status; var dStr = msg.toDartString(); @@ -97,7 +97,7 @@ class VLMInference { final status = vlmOV.vlmInferenceSetImagePaths(instance.ref.value, pointerToCStrings, cStrings.length); - if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) { + if (status.ref.status != StatusEnum.OkStatus) { throw "Close error: ${status.ref.status} ${status.ref.message.toDartString()}"; } vlmOV.freeStatus(status); @@ -106,7 +106,7 @@ class VLMInference { void forceStop() { final status = vlmOV.vlmInferenceStop(instance.ref.value); - if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) { + if (status.ref.status != StatusEnum.OkStatus) { throw "VLM Force Stop error: ${status.ref.status} ${status.ref.message.toDartString()}"; } } @@ -115,7 +115,7 @@ class VLMInference { void close() { final status = vlmOV.vlmInferenceClose(instance.ref.value); - if (StatusEnum.fromValue(status.ref.status) != StatusEnum.OkStatus) { + if (status.ref.status != StatusEnum.OkStatus) { throw "Close error: ${status.ref.status} ${status.ref.message.toDartString()}"; } vlmOV.freeStatus(status); diff --git a/lib/pages/computer_vision/widgets/camera_view.dart b/lib/pages/computer_vision/widgets/camera_view.dart index 6f8582cd..c5b653af 100644 --- a/lib/pages/computer_vision/widgets/camera_view.dart +++ b/lib/pages/computer_vision/widgets/camera_view.dart @@ -11,8 +11,11 @@ import 'package:fluent_ui/fluent_ui.dart'; import 'package:inference/annotation.dart'; import 'package:inference/interop/openvino_bindings.dart' show SerializationOutput; import 'package:inference/providers/image_inference_provider.dart'; +import 'package:inference/widgets/canvas/canvas.dart'; import 'package:provider/provider.dart'; +import 'dart:ui' as ui; + class CameraView extends StatefulWidget { final int deviceIndex; const CameraView({required this.deviceIndex, super.key}); @@ -32,7 +35,7 @@ class _CameraViewState extends State { void startCamera(){ streamController = StreamController(); - inferenceProvider.openCamera(widget.deviceIndex, onFrame, SerializationOutput(overlay: true)); + inferenceProvider.openCamera(widget.deviceIndex, onFrame, SerializationOutput(source: true, json: true)); } @override @@ -53,19 +56,19 @@ class _CameraViewState extends State { return StreamBuilder( stream: streamController.stream, builder: (context, AsyncSnapshot snapshot) { - print(snapshot); - final overlayData = snapshot.data?.overlay; - final overlayImage = overlayData == null - ? null - : Image.memory(base64Decode(overlayData), gaplessPlayback: true,); - - - return Builder( - builder: (context) { - if (overlayImage == null) { + Future? imageFuture = snapshot.data?.source != null ? decodeImageFromList(base64Decode(snapshot.data!.source!)) : null; + return FutureBuilder( + future: imageFuture, + builder: (context, imageSnapshot) { + if (imageSnapshot.data == null) { return Container(); } - return Center(child: overlayImage); + return Canvas( + image: imageSnapshot.data!, + annotations: snapshot.data!.parseAnnotations(), + labelDefinitions: inferenceProvider.project.labelDefinitions, + ); + } ); } diff --git a/lib/utils/image_graph_builder.dart b/lib/utils/image_graph_builder.dart index c44b8f02..17cf4293 100644 --- a/lib/utils/image_graph_builder.dart +++ b/lib/utils/image_graph_builder.dart @@ -275,6 +275,7 @@ class ImageGraphBuilder { input_stream : "INFERENCE_RESULT:inference_result" input_stream : "OVERLAY:overlay" input_stream : "OUTPUT:serialization_output" + input_stream : "SOURCE:input" output_stream: "RESULT:output" } """; diff --git a/lib/widgets/canvas/canvas_painter.dart b/lib/widgets/canvas/canvas_painter.dart index 42c79b47..f69ca378 100644 --- a/lib/widgets/canvas/canvas_painter.dart +++ b/lib/widgets/canvas/canvas_painter.dart @@ -153,7 +153,7 @@ class CanvasPainter extends CustomPainter { @override bool shouldRepaint(CanvasPainter oldDelegate) { - return false; + return oldDelegate.image != image; } @override bool shouldRebuildSemantics(CanvasPainter oldDelegate) => false; diff --git a/openvino_bindings/src/bindings.cc b/openvino_bindings/src/bindings.cc index 72f56f63..ca31fabd 100644 --- a/openvino_bindings/src/bindings.cc +++ b/openvino_bindings/src/bindings.cc @@ -310,23 +310,23 @@ Status* graphRunnerQueueImage(CGraphRunner instance, const char* name, int times } } -Status* graphRunnerQueueSerializationOutput(CGraphRunner instance, const char* name, int timestamp, bool json, bool csv, bool overlay) { +Status* graphRunnerQueueSerializationOutput(CGraphRunner instance, const char* name, int timestamp, bool json, bool csv, bool overlay, bool source) { try { - reinterpret_cast(instance)->queue(name, timestamp, SerializationOutput{json, csv, overlay}); + reinterpret_cast(instance)->queue(name, timestamp, SerializationOutput{json, csv, overlay, source}); return new Status{OkStatus, ""}; } catch (...) { return handle_exceptions(); } } -Status* graphRunnerStartCamera(CGraphRunner instance, int camera_index, ImageInferenceCallbackFunction callback, bool json, bool csv, bool overlay) { +Status* graphRunnerStartCamera(CGraphRunner instance, int camera_index, ImageInferenceCallbackFunction callback, bool json, bool csv, bool overlay, bool source) { try { auto runner = reinterpret_cast(instance); auto lambda_callback = [callback](std::string response) { callback(new StatusOrString{OkStatus, "", strdup(response.c_str())}); }; - runner->open_camera(camera_index, SerializationOutput{json, csv, overlay}, lambda_callback); + runner->open_camera(camera_index, SerializationOutput{json, csv, overlay, source}, lambda_callback); return new Status{OkStatus, ""}; } catch (...) { return handle_exceptions(); diff --git a/openvino_bindings/src/bindings.h b/openvino_bindings/src/bindings.h index 53d25437..1de714f1 100644 --- a/openvino_bindings/src/bindings.h +++ b/openvino_bindings/src/bindings.h @@ -197,10 +197,10 @@ EXPORT Status* vlmInferenceClose(CVLMInference instance); EXPORT StatusOrGraphRunner* graphRunnerOpen(const char* graph); EXPORT Status* graphRunnerQueueImage(CGraphRunner instance, const char* name, int timestamp, unsigned char* image_data, const size_t data_length); -EXPORT Status* graphRunnerQueueSerializationOutput(CGraphRunner instance, const char* name, int timestamp, bool json, bool csv, bool overlay); +EXPORT Status* graphRunnerQueueSerializationOutput(CGraphRunner instance, const char* name, int timestamp, bool json, bool csv, bool overlay, bool source); EXPORT StatusOrString* graphRunnerGet(CGraphRunner instance); EXPORT Status* graphRunnerStop(CGraphRunner instance); -EXPORT Status* graphRunnerStartCamera(CGraphRunner instance, int cameraIndex, ImageInferenceCallbackFunction callback, bool json, bool csv, bool overlay); +EXPORT Status* graphRunnerStartCamera(CGraphRunner instance, int cameraIndex, ImageInferenceCallbackFunction callback, bool json, bool csv, bool overlay, bool source); EXPORT StatusOrInt* graphRunnerGetTimestamp(CGraphRunner instance); EXPORT Status* graphRunnerStopCamera(CGraphRunner instance); diff --git a/openvino_bindings/src/mediapipe/serialization/serialization_calculators.cc b/openvino_bindings/src/mediapipe/serialization/serialization_calculators.cc index 47c24fff..efeb722d 100644 --- a/openvino_bindings/src/mediapipe/serialization/serialization_calculators.cc +++ b/openvino_bindings/src/mediapipe/serialization/serialization_calculators.cc @@ -18,6 +18,7 @@ namespace mediapipe { absl::Status SerializationCalculator::GetContract(CalculatorContract *cc) { LOG(INFO) << "SerializationCalculator::GetContract()"; + cc->Inputs().Tag("SOURCE").Set(); cc->Inputs().Tag("OVERLAY").Set().Optional(); cc->Inputs().Tag("OUTPUT").Set(); cc->Inputs().Tag("INFERENCE_RESULT").Set(); @@ -63,6 +64,11 @@ absl::Status SerializationCalculator::GetiProcess(CalculatorContext *cc) { output["overlay"] = geti::base64_encode_mat(overlay); } + if (selected_output.source) { + cv::Mat source = cc->Inputs().Tag("SOURCE").Get(); + output["source"] = geti::base64_encode_mat(source); + } + cc->Outputs() .Tag("RESULT") .AddPacket(MakePacket(output.dump()) diff --git a/openvino_bindings/src/mediapipe/serialization/serialization_calculators.h b/openvino_bindings/src/mediapipe/serialization/serialization_calculators.h index 1663858c..fe9c041b 100644 --- a/openvino_bindings/src/mediapipe/serialization/serialization_calculators.h +++ b/openvino_bindings/src/mediapipe/serialization/serialization_calculators.h @@ -23,6 +23,7 @@ class SerializationOutput { bool json; bool csv; bool overlay; + bool source; }; namespace mediapipe { From 16910c8648e94452afe5f8aa0e8b455b79906685 Mon Sep 17 00:00:00 2001 From: Ronald Hecker Date: Mon, 28 Apr 2025 12:11:21 +0200 Subject: [PATCH 05/12] Imported models are automatically npu enabled for now --- lib/deployment_processor.dart | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/deployment_processor.dart b/lib/deployment_processor.dart index 011fe699..5c4557a2 100644 --- a/lib/deployment_processor.dart +++ b/lib/deployment_processor.dart @@ -61,6 +61,9 @@ Future> loadProjectsFromStorage() async { projectFile.writeAsStringSync(encoder.convert(jsonContent)); } final project = Project.fromJson(jsonContent, projectFolder); + if (Config().externalModels.contains(projectFolder) && project is PublicProject) { + project.manifest.npuEnabled = true; + } project.loaded.complete(); return project; } catch (exception, stack) { From 4cb20ee32e23ab89d83a56f2135be79d5fe171de Mon Sep 17 00:00:00 2001 From: Ronald Hecker Date: Wed, 30 Apr 2025 13:34:14 +0200 Subject: [PATCH 06/12] Adds resolution selection for camera inference Previously it would show default (usually 640x480) Also added fps counter --- lib/interop/device.dart | 17 +- lib/interop/generated_bindings.dart | 33 ++++ lib/interop/graph_runner.dart | 13 ++ lib/pages/computer_vision/live_inference.dart | 2 +- .../computer_vision/widgets/camera_view.dart | 181 +++++++++++++++--- lib/providers/image_inference_provider.dart | 5 + lib/widgets/canvas/canvas.dart | 12 ++ openvino_bindings/src/bindings.cc | 21 +- openvino_bindings/src/bindings.h | 9 + .../src/mediapipe/graph_runner.cc | 6 + .../src/mediapipe/graph_runner.h | 2 + openvino_bindings/src/utils/camera_handler.cc | 14 +- openvino_bindings/src/utils/camera_handler.h | 3 + openvino_bindings/src/utils/input_devices.h | 66 ++++++- pubspec.lock | 2 +- pubspec.yaml | 5 +- 16 files changed, 353 insertions(+), 38 deletions(-) diff --git a/lib/interop/device.dart b/lib/interop/device.dart index 5b25bf3d..88547cc5 100644 --- a/lib/interop/device.dart +++ b/lib/interop/device.dart @@ -39,10 +39,18 @@ class Device { } } +class Resolution { + final int width; + final int height; + + const Resolution(this.width, this.height); +} + class CameraDevice { final int id; final String name; - const CameraDevice(this.id, this.name); + final List resolutions; + const CameraDevice(this.id, this.name, this.resolutions); static Future> getDevices() async { final result = await Isolate.run(() { @@ -54,9 +62,14 @@ class CameraDevice { List devices = []; for (int i = 0; i < status.ref.size; i++) { + List resolutions = []; + for (int j = 0; j < status.ref.value[i].size; j++ ){ + resolutions.add(Resolution(status.ref.value[i].resolutions[j].width, status.ref.value[i].resolutions[j].height)); + } devices.add(CameraDevice( status.ref.value[i].id, - status.ref.value[i].name.toDartString() + status.ref.value[i].name.toDartString(), + resolutions )); } deviceOV.freeStatusOrCameraDevices(status); diff --git a/lib/interop/generated_bindings.dart b/lib/interop/generated_bindings.dart index 7127cbec..56aa7525 100644 --- a/lib/interop/generated_bindings.dart +++ b/lib/interop/generated_bindings.dart @@ -654,6 +654,26 @@ class OpenVINO { late final _graphRunnerStopCamera = _graphRunnerStopCameraPtr .asFunction Function(CGraphRunner)>(); + ffi.Pointer graphRunnerSetCameraResolution( + CGraphRunner instance, + int width, + int height, + ) { + return _graphRunnerSetCameraResolution( + instance, + width, + height, + ); + } + + late final _graphRunnerSetCameraResolutionPtr = _lookup< + ffi.NativeFunction< + ffi.Pointer Function(CGraphRunner, ffi.Int, + ffi.Int)>>('graphRunnerSetCameraResolution'); + late final _graphRunnerSetCameraResolution = + _graphRunnerSetCameraResolutionPtr + .asFunction Function(CGraphRunner, int, int)>(); + ffi.Pointer sentenceTransformerOpen( ffi.Pointer model_path, ffi.Pointer device, @@ -1014,11 +1034,24 @@ final class Device extends ffi.Struct { external ffi.Pointer name; } +final class CameraResolution extends ffi.Struct { + @ffi.Int() + external int width; + + @ffi.Int() + external int height; +} + final class CameraDevice extends ffi.Struct { @ffi.Int() external int id; external ffi.Pointer name; + + external ffi.Pointer resolutions; + + @ffi.Int() + external int size; } final class TranscriptionChunk extends ffi.Struct { diff --git a/lib/interop/graph_runner.dart b/lib/interop/graph_runner.dart index b7133e20..0f0e4028 100644 --- a/lib/interop/graph_runner.dart +++ b/lib/interop/graph_runner.dart @@ -10,6 +10,7 @@ import 'dart:isolate'; import 'dart:typed_data'; import 'package:ffi/ffi.dart'; +import 'package:inference/interop/device.dart'; import 'package:inference/interop/openvino_bindings.dart'; final ov = getBindings(); @@ -60,6 +61,18 @@ class GraphRunner { }); } + Future setCameraResolution(Resolution resolution) async { + int instanceAddress = instance.ref.value.address; + return await Isolate.run(() { + final status = ov.graphRunnerSetCameraResolution(Pointer.fromAddress(instanceAddress), resolution.width, resolution.height); + + if (status.ref.status != StatusEnum.OkStatus) { + throw "GraphRunner::setCameraResolution error: ${status.ref.status} ${status.ref.message.toDartString()}"; + } + ov.freeStatus(status); + }); + } + Future startCamera(int deviceIndex, Function(String) callback, SerializationOutput output) async { void wrapCallback(Pointer ptr) { if (ptr.ref.status != StatusEnum.OkStatus) { diff --git a/lib/pages/computer_vision/live_inference.dart b/lib/pages/computer_vision/live_inference.dart index be2af4d4..b01a3db7 100644 --- a/lib/pages/computer_vision/live_inference.dart +++ b/lib/pages/computer_vision/live_inference.dart @@ -174,7 +174,7 @@ class _LiveInferenceState extends State { builder: (context, snapshot) { if (snapshot.connectionState == ConnectionState.done) { return switch(mode) { - LiveInferenceMode.camera => CameraView(deviceIndex: cameraDevice!.id), + LiveInferenceMode.camera => CameraView(device: cameraDevice!), LiveInferenceMode.image => DropArea( type: "image", showChild: inferenceResult != null, diff --git a/lib/pages/computer_vision/widgets/camera_view.dart b/lib/pages/computer_vision/widgets/camera_view.dart index c5b653af..c7315567 100644 --- a/lib/pages/computer_vision/widgets/camera_view.dart +++ b/lib/pages/computer_vision/widgets/camera_view.dart @@ -7,18 +7,53 @@ import 'dart:async'; import 'dart:convert'; +import 'package:intl/intl.dart'; +import 'package:rxdart/rxdart.dart'; + import 'package:fluent_ui/fluent_ui.dart'; import 'package:inference/annotation.dart'; +import 'package:inference/interop/device.dart'; import 'package:inference/interop/openvino_bindings.dart' show SerializationOutput; import 'package:inference/providers/image_inference_provider.dart'; +import 'package:inference/theme_fluent.dart'; import 'package:inference/widgets/canvas/canvas.dart'; +import 'package:inference/widgets/controls/no_outline_button.dart'; +import 'package:inference/widgets/grid_container.dart'; import 'package:provider/provider.dart'; import 'dart:ui' as ui; + +class SmoothFPSCounter { + late final StreamController _fpsStreamController; + late final Stream fpsStream; + + int counter = 0; + late DateTime start; + + SmoothFPSCounter(Stream stream){ + _fpsStreamController = StreamController(); + fpsStream = _fpsStreamController.stream.asBroadcastStream(); + reset(); + + stream.listen((_) { + counter += 1; + }); + stream.throttleTime(Duration(seconds: 1)).listen((p) { + final fps = counter.toDouble() / DateTime.now().difference(start).inMilliseconds * 1000.0; + _fpsStreamController.add(fps); + }); + } + + void reset() { + counter = 0; + start = DateTime.now(); + } +} + class CameraView extends StatefulWidget { - final int deviceIndex; - const CameraView({required this.deviceIndex, super.key}); + final CameraDevice device; + const CameraView({required this.device, super.key}); @override State createState() => _CameraViewState(); @@ -27,20 +62,24 @@ class CameraView extends StatefulWidget { class _CameraViewState extends State { late StreamController streamController; + late final Stream stream; late ImageInferenceProvider inferenceProvider; + late SmoothFPSCounter fpsCounter; void onFrame(ImageInferenceResult result) { streamController.add(result); } void startCamera(){ - streamController = StreamController(); - inferenceProvider.openCamera(widget.deviceIndex, onFrame, SerializationOutput(source: true, json: true)); + inferenceProvider.openCamera(widget.device.id, onFrame, SerializationOutput(source: true, json: true)); } @override void initState() { super.initState(); + streamController = StreamController(); + stream = streamController.stream.asBroadcastStream(); + fpsCounter = SmoothFPSCounter(stream); inferenceProvider = Provider.of(context, listen: false); startCamera(); } @@ -53,25 +92,123 @@ class _CameraViewState extends State { @override Widget build(BuildContext context) { - return StreamBuilder( - stream: streamController.stream, - builder: (context, AsyncSnapshot snapshot) { - Future? imageFuture = snapshot.data?.source != null ? decodeImageFromList(base64Decode(snapshot.data!.source!)) : null; - return FutureBuilder( - future: imageFuture, - builder: (context, imageSnapshot) { - if (imageSnapshot.data == null) { - return Container(); + return Column( + children: [ + CameraOptions( + fpsStream: fpsCounter.fpsStream, + device: widget.device, + inferenceProvider: inferenceProvider, + onResolutionChange: (_) { + fpsCounter.reset(); + }, + ), + Expanded( + child: StreamBuilder( + stream: stream, + builder: (context, AsyncSnapshot snapshot) { + Future? imageFuture = snapshot.data?.source != null ? decodeImageFromList(base64Decode(snapshot.data!.source!)) : null; + return FutureBuilder( + future: imageFuture, + builder: (context, imageSnapshot) { + if (imageSnapshot.data == null) { + return Container(); + } + return Canvas( + image: imageSnapshot.data!, + annotations: snapshot.data!.parseAnnotations(), + labelDefinitions: inferenceProvider.project.labelDefinitions, + ); + + } + ); } - return Canvas( - image: imageSnapshot.data!, - annotations: snapshot.data!.parseAnnotations(), - labelDefinitions: inferenceProvider.project.labelDefinitions, - ); - - } - ); - } + ), + ), + ], + ); + } +} + +class CameraOptions extends StatefulWidget { + final CameraDevice device; + final ImageInferenceProvider inferenceProvider; + final Stream fpsStream; + final Function(Resolution resolution)? onResolutionChange; + + const CameraOptions({ + required this.device, + required this.inferenceProvider, + required this.fpsStream, + this.onResolutionChange, + super.key}); + + @override + State createState() => _CameraOptionsState(); +} + +class _CameraOptionsState extends State { + + late Resolution resolution; + + @override + void initState() { + super.initState(); + resolution = widget.device.resolutions.first; + } + + void setResolution(Resolution res, ImageInferenceProvider inferenceProvider) { + setState(() { + inferenceProvider.setCameraResolution(res); + resolution = res; + widget.onResolutionChange?.call(res); + }); + } + + @override + Widget build(BuildContext context) { + final theme = FluentTheme.of(context); + final fpsFormatter = NumberFormat.decimalPatternDigits(decimalDigits: 2); + return SizedBox( + height: 64, + child: GridContainer( + color: neutralBackground.of(theme), + padding: const EdgeInsets.symmetric(horizontal: 16), + child: Row( + children: [ + StreamBuilder( + stream: widget.fpsStream, + builder: (context, snapshot) { + final fps = snapshot.data ?? 0; + return Text("FPS: ${fpsFormatter.format(fps)}"); + } + ), + Builder( + builder: (context) { + return DropDownButton( + buttonBuilder: (context, callback) { + return NoOutlineButton( + onPressed: callback, + child: Row( + children: [ + Text("Resolution: ${resolution.width} x ${resolution.height}"), + const Padding( + padding: EdgeInsets.only(left: 8), + child: Icon(FluentIcons.chevron_down, size: 12), + ), + ], + ), + ); + }, + items: [ + for (final resolution in widget.device.resolutions) + MenuFlyoutItem(text: Text("${resolution.width} x ${resolution.height}"), onPressed: () => setResolution(resolution, widget.inferenceProvider)), + ] + ); + } + ), + ], + ), + ), ); } } diff --git a/lib/providers/image_inference_provider.dart b/lib/providers/image_inference_provider.dart index 37474144..4eca2b1d 100644 --- a/lib/providers/image_inference_provider.dart +++ b/lib/providers/image_inference_provider.dart @@ -7,6 +7,7 @@ import 'dart:convert'; import 'package:flutter/foundation.dart'; import 'package:inference/annotation.dart'; +import 'package:inference/interop/device.dart'; import 'package:inference/utils/image_graph_builder.dart'; import 'package:inference/interop/graph_runner.dart'; import 'package:inference/interop/openvino_bindings.dart'; @@ -69,6 +70,10 @@ class ImageInferenceProvider extends ChangeNotifier { super.dispose(); } + void setCameraResolution(Resolution resolution) { + _inference!.setCameraResolution(resolution); + } + void openCamera(int deviceIndex, Function(ImageInferenceResult) callback, SerializationOutput output) { _inference!.startCamera(deviceIndex, (String output) => callback(ImageInferenceResult.fromJson(jsonDecode(output))), output); } diff --git a/lib/widgets/canvas/canvas.dart b/lib/widgets/canvas/canvas.dart index 344adda0..cddbf47d 100644 --- a/lib/widgets/canvas/canvas.dart +++ b/lib/widgets/canvas/canvas.dart @@ -13,6 +13,7 @@ import 'package:inference/widgets/canvas/canvas_painter.dart'; import 'package:vector_math/vector_math_64.dart' show Vector3; import 'package:inference/project.dart' as project; + class Canvas extends StatefulWidget { final ui.Image image; @@ -33,6 +34,7 @@ class _CanvasState extends State { Matrix4 inverse = Matrix4.identity(); bool done = false; + @override void initState() { super.initState(); @@ -74,6 +76,16 @@ class _CanvasState extends State { }); } + @override + void didUpdateWidget(covariant Canvas oldWidget) { + super.didUpdateWidget(oldWidget); + if (oldWidget.image.width != widget.image.width || oldWidget.image.height != widget.image.height) { + Future.delayed(Duration.zero, () { + matrix = setTransformToFit(widget.image); + }); + } + } + @override Widget build(BuildContext context) { return NotificationListener( diff --git a/openvino_bindings/src/bindings.cc b/openvino_bindings/src/bindings.cc index ca31fabd..5d0d8077 100644 --- a/openvino_bindings/src/bindings.cc +++ b/openvino_bindings/src/bindings.cc @@ -80,6 +80,10 @@ void freeStatusOrEmbeddings(StatusOrEmbeddings *status) { void freeStatusOrCameraDevices(StatusOrCameraDevices *status) { if (status->status == StatusEnum::OkStatus) { + for (int i = 0; i < status->size; i++) { + delete [] status->value[i].resolutions; + status->value[i].resolutions = NULL; + } delete [] status->value; status->value = NULL; // Prevent dangling pointers } @@ -333,6 +337,15 @@ Status* graphRunnerStartCamera(CGraphRunner instance, int camera_index, ImageInf } } +Status* graphRunnerSetCameraResolution(CGraphRunner instance, int width, int height) { + try { + reinterpret_cast(instance)->set_camera_resolution(width, height); + return new Status{OkStatus, ""}; + } catch (...) { + return handle_exceptions(); + } +} + StatusOrInt* graphRunnerGetTimestamp(CGraphRunner instance) { try { auto graph_runner = reinterpret_cast(instance); @@ -352,6 +365,7 @@ Status* graphRunnerStopCamera(CGraphRunner instance) { } } + StatusOrString* graphRunnerGet(CGraphRunner instance) { try { auto graph_runner = reinterpret_cast(instance); @@ -477,7 +491,12 @@ StatusOrCameraDevices* getAvailableCameraDevices() { CameraDevice* devices = new CameraDevice[cameras.size()]; int i = 0; for (auto camera: cameras) { - devices[i] = { (int)camera.first, strdup(camera.second.c_str()) }; + int j = 0; + devices[i] = { (int)camera.id, strdup(camera.name.c_str()), new CameraResolution[camera.resolutions.size()], (int)camera.resolutions.size()}; + for (auto resolution: camera.resolutions) { + devices[i].resolutions[j] = {resolution.width, resolution.height}; + j++; + } i++; } diff --git a/openvino_bindings/src/bindings.h b/openvino_bindings/src/bindings.h index 1de714f1..5d75d4b4 100644 --- a/openvino_bindings/src/bindings.h +++ b/openvino_bindings/src/bindings.h @@ -37,9 +37,16 @@ typedef struct { const char* name; } Device; +typedef struct { + int width; + int height; +} CameraResolution; + typedef struct { int id; const char* name; + CameraResolution* resolutions; + int size; } CameraDevice; typedef struct { @@ -203,6 +210,8 @@ EXPORT Status* graphRunnerStop(CGraphRunner instance); EXPORT Status* graphRunnerStartCamera(CGraphRunner instance, int cameraIndex, ImageInferenceCallbackFunction callback, bool json, bool csv, bool overlay, bool source); EXPORT StatusOrInt* graphRunnerGetTimestamp(CGraphRunner instance); EXPORT Status* graphRunnerStopCamera(CGraphRunner instance); +EXPORT Status* graphRunnerSetCameraResolution(CGraphRunner instance, int width, int height); + EXPORT StatusOrSentenceTransformer* sentenceTransformerOpen(const char* model_path, const char* device); EXPORT StatusOrEmbeddings* sentenceTransformerGenerate(CSentenceTransformer instance, const char* prompt); diff --git a/openvino_bindings/src/mediapipe/graph_runner.cc b/openvino_bindings/src/mediapipe/graph_runner.cc index d9b5c54f..4d963393 100644 --- a/openvino_bindings/src/mediapipe/graph_runner.cc +++ b/openvino_bindings/src/mediapipe/graph_runner.cc @@ -36,6 +36,12 @@ void GraphRunner::stop() { } +void GraphRunner::set_camera_resolution(int width, int height) { + if (camera_handler) { + camera_handler->set_resolution(width, height); + } +} + void GraphRunner::open_camera(int deviceIndex, SerializationOutput serializationOutput, const std::function& callback) { camera_handler = std::make_shared(deviceIndex); diff --git a/openvino_bindings/src/mediapipe/graph_runner.h b/openvino_bindings/src/mediapipe/graph_runner.h index 5cb5fb6c..26c56dae 100644 --- a/openvino_bindings/src/mediapipe/graph_runner.h +++ b/openvino_bindings/src/mediapipe/graph_runner.h @@ -38,6 +38,8 @@ class GraphRunner { std::string get(); void stop(); + + void set_camera_resolution(int width, int height); void open_camera(int deviceIndex, SerializationOutput serializationOutput, const std::function& callback); void stop_camera(); diff --git a/openvino_bindings/src/utils/camera_handler.cc b/openvino_bindings/src/utils/camera_handler.cc index f8f51221..632761f5 100644 --- a/openvino_bindings/src/utils/camera_handler.cc +++ b/openvino_bindings/src/utils/camera_handler.cc @@ -18,11 +18,19 @@ void CameraHandler::stop_camera() { } } +void CameraHandler::set_resolution(int width, int height) { + if (width > 0) { + cap.set(cv::CAP_PROP_FRAME_WIDTH, width); + } + if (height > 0) { + cap.set(cv::CAP_PROP_FRAME_HEIGHT, height); + } +} void CameraHandler::start_camera_process(const std::function& onFrameCallback) { - cv::VideoCapture cap; + cap = cv::VideoCapture(device); + std::cout << "opening device: " << std::endl; std::cout << device << std::endl; - cap.open(device); if (!cap.isOpened()) { throw api_error(CameraNotOpenend); } @@ -31,7 +39,7 @@ void CameraHandler::start_camera_process(const std::function& onFrameCallback); void stop_camera(); + void set_resolution(int width, int height); private: void start_camera_process(const std::function& onFrameCallback); bool camera_get_frame = false; + + cv::VideoCapture cap; std::thread camera_thread; diff --git a/openvino_bindings/src/utils/input_devices.h b/openvino_bindings/src/utils/input_devices.h index c94c2ae2..2ab2ff7e 100644 --- a/openvino_bindings/src/utils/input_devices.h +++ b/openvino_bindings/src/utils/input_devices.h @@ -25,7 +25,31 @@ #pragma comment(lib, "mfplat.lib") #pragma comment(lib, "mf.lib") -std::map list_camera_devices() { +class Resolution { +public: + int width; + int height; +}; + +class Camera { +public: + size_t id; + std::string name; + std::vector resolutions; + + bool add_resolution(Resolution resolution) { + for (auto &rhs: resolutions) { + if (rhs.width == resolution.width && rhs.height == resolution.height) { + return false; + } + } + resolutions.push_back(resolution); + return true; + } + +}; + +std::vector list_camera_devices() { HRESULT hr = MFStartup(MF_VERSION); if (FAILED(hr)) { throw api_error(InputDeviceError, "MFStartup failed."); @@ -50,7 +74,7 @@ std::map list_camera_devices() { UINT32 count = 0; - std::map cameras = {}; + std::vector cameras = {}; hr = MFEnumDeviceSources(pAttributes, &ppDevices, &count); if (SUCCEEDED(hr)) { for (UINT32 i = 0; i < count; i++) { @@ -58,8 +82,38 @@ std::map list_camera_devices() { UINT32 cchName = 0; hr = ppDevices[i]->GetAllocatedString(MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME, &szFriendlyName, &cchName); if (SUCCEEDED(hr)) { + IMFMediaSource* pSource = nullptr; + ppDevices[i]->ActivateObject(IID_PPV_ARGS(&pSource)); + + IMFPresentationDescriptor* pPD = nullptr; + pSource->CreatePresentationDescriptor(&pPD); + + IMFStreamDescriptor* pSD = nullptr; + BOOL selected; + pPD->GetStreamDescriptorByIndex(0, &selected, &pSD); + + IMFMediaTypeHandler* pHandler = nullptr; + pSD->GetMediaTypeHandler(&pHandler); + + std::wstring ws(szFriendlyName); - cameras.insert({i, std::string(ws.begin(), ws.end())}); + auto camera = Camera{i, std::string(ws.begin(), ws.end()), {}}; + + DWORD mediaTypeCount = 0; + pHandler->GetMediaTypeCount(&mediaTypeCount); + + + for (DWORD i = 0; i < mediaTypeCount; i++) { + IMFMediaType* pType = nullptr; + pHandler->GetMediaTypeByIndex(i, &pType); + + UINT32 width = 0, height = 0; + MFGetAttributeSize(pType, MF_MT_FRAME_SIZE, &width, &height); + camera.add_resolution(Resolution{(int)width, (int)height}); + pType->Release(); + } + + cameras.push_back(camera); //std::wcout << L"[" << i << L"]: " << szFriendlyName << std::endl; CoTaskMemFree(szFriendlyName); } @@ -76,7 +130,7 @@ std::map list_camera_devices() { return cameras; } #elif __APPLE__ -std::map list_camera_devices() { +std::vector list_camera_devices() { return {}; } #elif __linux__ @@ -85,7 +139,7 @@ std::map list_camera_devices() { #include #include -std::map list_camera_devices() { +std::vector list_camera_devices() { std::map cameras = {}; for (int i = 0; i < 10; ++i) { // Check up to 10 devices @@ -97,7 +151,7 @@ std::map list_camera_devices() { struct v4l2_capability cap; if (ioctl(fd, VIDIOC_QUERYCAP, &cap) == 0) { if (cap.capabilities & V4L2_CAP_VIDEO_CAPTURE) { - cameras.insert({i, std::string(reinterpret_cast(cap.card))}); + cameras.push_back(Camera{i, std::string(reinterpret_cast(cap.card)), {}}); } } close(fd); diff --git a/pubspec.lock b/pubspec.lock index ec578395..48e2608b 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -938,7 +938,7 @@ packages: source: hosted version: "4.1.0" rxdart: - dependency: transitive + dependency: "direct main" description: name: rxdart sha256: "5c3004a4a8dbb94bd4bf5412a4def4acdaa12e12f269737a5751369e12d1a962" diff --git a/pubspec.yaml b/pubspec.yaml index f4050593..33bf64af 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -73,6 +73,7 @@ dependencies: jinja: ^0.6.0 docx_to_text: ^1.0.1 dart_pdf_reader: ^2.1.0 + rxdart: ^0.28.0 dev_dependencies: flutter_test: @@ -180,9 +181,9 @@ msix_config: display_name: OpenVINO Test Drive publisher_display_name: Intel identity_name: intel.openvino.testdrive - msix_version: 25.0.0.5 + msix_version: 25.1.0.2 trim_logo: false - output_name: OpenVINO Test Drive 25.0.0 + output_name: OpenVINO Test Drive 25.1.0 capabilities: "internetClient" logo_path: "images/logo_250.png" app_installer: #<-- app installer configuration From 50e6d3b251bb0c6ee7c756035cb4684d15c225ae Mon Sep 17 00:00:00 2001 From: Ronald Hecker Date: Thu, 8 May 2025 13:07:06 +0200 Subject: [PATCH 07/12] Rename label for vlm metrics Openvino 25.1.0 does not output proper vlm metrics as far as I can see --- lib/pages/vlm/widgets/vlm_metrics_grid.dart | 2 +- pubspec.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pages/vlm/widgets/vlm_metrics_grid.dart b/lib/pages/vlm/widgets/vlm_metrics_grid.dart index d406cb4a..beaf971a 100644 --- a/lib/pages/vlm/widgets/vlm_metrics_grid.dart +++ b/lib/pages/vlm/widgets/vlm_metrics_grid.dart @@ -29,7 +29,7 @@ class VLMMetricsGrid extends StatelessWidget { unit: "ms", ), MetricsCard( - header: "Time to generate image", + header: "Time to generate answer", value: nf.format(metrics.generate_time), unit: "ms", ) diff --git a/pubspec.yaml b/pubspec.yaml index 33bf64af..57821a1c 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -181,7 +181,7 @@ msix_config: display_name: OpenVINO Test Drive publisher_display_name: Intel identity_name: intel.openvino.testdrive - msix_version: 25.1.0.2 + msix_version: 25.1.0.3 trim_logo: false output_name: OpenVINO Test Drive 25.1.0 capabilities: "internetClient" From 2de627a9a10705aba6e4986fd4f20be8da235f3d Mon Sep 17 00:00:00 2001 From: Ronald Hecker Date: Wed, 14 May 2025 14:39:13 +0200 Subject: [PATCH 08/12] Implement mqtt automation --- .../widgets/automation_options.dart | 276 ++++++++++++++++++ .../computer_vision/widgets/camera_view.dart | 103 ++++--- pubspec.lock | 16 + pubspec.yaml | 3 +- 4 files changed, 351 insertions(+), 47 deletions(-) create mode 100644 lib/pages/computer_vision/widgets/automation_options.dart diff --git a/lib/pages/computer_vision/widgets/automation_options.dart b/lib/pages/computer_vision/widgets/automation_options.dart new file mode 100644 index 00000000..89b20f1c --- /dev/null +++ b/lib/pages/computer_vision/widgets/automation_options.dart @@ -0,0 +1,276 @@ +import 'dart:convert'; + +import 'package:fluent_ui/fluent_ui.dart'; +import 'package:flutter/services.dart'; +import 'package:inference/annotation.dart'; +import 'package:mqtt_client/mqtt_client.dart'; +import 'package:mqtt_client/mqtt_server_client.dart'; + +class MqttClientWrapper { + final MqttServerClient client; + final String topic; + final bool retain; + + const MqttClientWrapper({required this.client, required this.topic, required this.retain}); + + void publish(String data) { + if (client.connectionStatus?.state == MqttConnectionState.connected){ + final builder = MqttClientPayloadBuilder(); + builder.addString(data); + client.publishMessage(topic, MqttQos.exactlyOnce, builder.payload!, retain: retain); + } + } + + void disconnect() { + client.disconnect(); + } +} + +class AutomationOptions extends StatefulWidget { + final Stream stream; + + const AutomationOptions({required this.stream, super.key}); + + @override + State createState() => _AutomationOptionsState(); +} + +class _AutomationOptionsState extends State { + + MqttClientWrapper? client; + + void publishToMqtt(ImageInferenceResult result) { + if (client != null && result.json != null) { + client!.publish(jsonEncode(result.json!)); + } + } + + @override + void initState() { + super.initState(); + widget.stream.listen(publishToMqtt); + } + + @override + void dispose() { + super.dispose(); + client?.disconnect(); + } + + @override + Widget build(BuildContext context) { + return Row( + children: [ + OutlinedButton( + style: ButtonStyle( + shape:WidgetStatePropertyAll(RoundedRectangleBorder( + borderRadius: BorderRadius.circular(4.0), + side: const BorderSide(color: Color(0XFF545454)), + )), + ), + onPressed: () async { + print("open dialog"); + if (client != null) { + client?.disconnect(); + setState(() { + client = null; + }); + } else { + final newClient = await showDialog( + context: context, + builder: (context) => AutomationOptionsDialog() + ); + + print("New client: $newClient"); + + setState(() { + client = newClient; + }); + } + }, + child: Text((client == null ? "Connect to MQTT Broker": "Disconnect")), + ), + ], + ); + } +} + +class AutomationOptionsDialog extends StatefulWidget { + const AutomationOptionsDialog({super.key}); + + @override + State createState() => _AutomationOptionsDialogState(); +} + +class _AutomationOptionsDialogState extends State { + final TextEditingController _hostController = TextEditingController(text: ""); + final TextEditingController _portController = TextEditingController(text: "1883"); + final TextEditingController _usernameController = TextEditingController(text: ""); + final TextEditingController _passwordController = TextEditingController(text: ""); + final TextEditingController _topicController = TextEditingController(text: ""); + + bool retain = false; + bool loading = false; + + List errors = []; + + void connect() async { + final client = MqttServerClient.withPort(_hostController.text, "test drive", int.parse(_portController.text)); + setState(() { loading = true; errors = []; }); + client.connect(_usernameController.text, _passwordController.text) + .then((status) { + setState(() { loading = false; }); + print(status.toString()); + if (status?.state == MqttConnectionState.connected) { + if (mounted) { + Navigator.pop(context, MqttClientWrapper(client: client, retain: retain, topic: _topicController.text)); + return; + } + } + MqttUtilities.asyncSleep(3).then((_) => client.disconnect); // ensure the client is disconnected on error + }) + .onError((e, stacktrace) { + setState(() { + errors.add(e.toString()); + loading = false; + }); + print("Error: $e: \n $stacktrace") ; + }); + } + + @override + void dispose() { + super.dispose(); + + _hostController.dispose(); + _portController.dispose(); + _usernameController.dispose(); + _passwordController.dispose(); + _topicController.dispose(); + } + + @override + Widget build(BuildContext context) { + return ContentDialog( + constraints: const BoxConstraints( + maxWidth: 756, + maxHeight: 500, + ), + title: const Text('Connect to MQTT Broker'), + content: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Expanded( + child: Column( + children: [ + Row( + mainAxisAlignment: MainAxisAlignment.spaceBetween, + children: [ + SizedBox( + width: 600, + child: InfoLabel( + label: "Host", + child: TextBox( + placeholder: "", + controller: _hostController, + onChanged: (_) {}, + ) + ), + ), + SizedBox( + width: 100, + child: InfoLabel( + label: "Port", + child: TextBox( + keyboardType: TextInputType.number, + inputFormatters: [FilteringTextInputFormatter.digitsOnly], + placeholder: "", + controller: _portController, + onChanged: (_) {}, + ) + ), + ), + ], + ), + SizedBox(height: 10), + InfoLabel( + label: "Username", + child: TextBox( + placeholder: "", + controller: _usernameController, + onChanged: (_) {}, + ) + ), + SizedBox(height: 10), + InfoLabel( + label: "Password", + child: TextBox( + placeholder: "", + controller: _passwordController, + onChanged: (_) {}, + ) + ), + SizedBox(height: 10), + Row( + mainAxisAlignment: MainAxisAlignment.spaceBetween, + children: [ + SizedBox( + width: 600, + child: InfoLabel( + label: "Topic", + child: TextBox( + placeholder: "", + controller: _topicController, + onChanged: (_) {}, + ) + ), + ), + SizedBox( + width: 100, + child: InfoLabel( + label: "Retain", + child: ToggleSwitch( + onChanged: (v) { setState(() { retain = v; });}, + checked: retain, + ), + ), + ), + + ], + ) + ], + ), + ), + Column( + children: [ + for (var error in errors) Text(error) + ] + ), + Row( + mainAxisAlignment: MainAxisAlignment.spaceBetween, + children: [ + SizedBox( + width: 500, + child: (loading ? ProgressBar() : Container()) + ), + Row( + mainAxisAlignment: MainAxisAlignment.end, + children: [ + Button( + onPressed: () => Navigator.pop(context), + child: const Text("Cancel"), + ), + const SizedBox(width: 10), + FilledButton( + onPressed: connect, + child: const Text("Connect"), + ), + ], + ), + ], + ) + ], + ) + ); + } +} diff --git a/lib/pages/computer_vision/widgets/camera_view.dart b/lib/pages/computer_vision/widgets/camera_view.dart index c7315567..e4b4c3c2 100644 --- a/lib/pages/computer_vision/widgets/camera_view.dart +++ b/lib/pages/computer_vision/widgets/camera_view.dart @@ -7,6 +7,7 @@ import 'dart:async'; import 'dart:convert'; +import 'package:inference/pages/computer_vision/widgets/automation_options.dart'; import 'package:intl/intl.dart'; import 'package:rxdart/rxdart.dart'; @@ -92,15 +93,31 @@ class _CameraViewState extends State { @override Widget build(BuildContext context) { + final theme = FluentTheme.of(context); return Column( children: [ - CameraOptions( - fpsStream: fpsCounter.fpsStream, - device: widget.device, - inferenceProvider: inferenceProvider, - onResolutionChange: (_) { - fpsCounter.reset(); - }, + SizedBox( + height: 64, + child: GridContainer( + color: neutralBackground.of(theme), + padding: const EdgeInsets.symmetric(horizontal: 16), + child: Row( + mainAxisAlignment: MainAxisAlignment.spaceBetween, + children: [ + CameraOptions( + fpsStream: fpsCounter.fpsStream, + device: widget.device, + inferenceProvider: inferenceProvider, + onResolutionChange: (_) { + fpsCounter.reset(); + }, + ), + AutomationOptions( + stream: stream + ), + ], + ), + ), ), Expanded( child: StreamBuilder( @@ -129,6 +146,8 @@ class _CameraViewState extends State { } } + + class CameraOptions extends StatefulWidget { final CameraDevice device; final ImageInferenceProvider inferenceProvider; @@ -166,49 +185,41 @@ class _CameraOptionsState extends State { @override Widget build(BuildContext context) { - final theme = FluentTheme.of(context); final fpsFormatter = NumberFormat.decimalPatternDigits(decimalDigits: 2); - return SizedBox( - height: 64, - child: GridContainer( - color: neutralBackground.of(theme), - padding: const EdgeInsets.symmetric(horizontal: 16), - child: Row( - children: [ - StreamBuilder( - stream: widget.fpsStream, - builder: (context, snapshot) { - final fps = snapshot.data ?? 0; - return Text("FPS: ${fpsFormatter.format(fps)}"); - } - ), - Builder( - builder: (context) { - return DropDownButton( - buttonBuilder: (context, callback) { - return NoOutlineButton( - onPressed: callback, - child: Row( - children: [ - Text("Resolution: ${resolution.width} x ${resolution.height}"), - const Padding( - padding: EdgeInsets.only(left: 8), - child: Icon(FluentIcons.chevron_down, size: 12), - ), - ], + return Row( + children: [ + StreamBuilder( + stream: widget.fpsStream, + builder: (context, snapshot) { + final fps = snapshot.data ?? 0; + return Text("FPS: ${fpsFormatter.format(fps)}"); + } + ), + Builder( + builder: (context) { + return DropDownButton( + buttonBuilder: (context, callback) { + return NoOutlineButton( + onPressed: callback, + child: Row( + children: [ + Text("Resolution: ${resolution.width} x ${resolution.height}"), + const Padding( + padding: EdgeInsets.only(left: 8), + child: Icon(FluentIcons.chevron_down, size: 12), ), - ); - }, - items: [ - for (final resolution in widget.device.resolutions) - MenuFlyoutItem(text: Text("${resolution.width} x ${resolution.height}"), onPressed: () => setResolution(resolution, widget.inferenceProvider)), - ] + ], + ), ); - } - ), - ], + }, + items: [ + for (final resolution in widget.device.resolutions) + MenuFlyoutItem(text: Text("${resolution.width} x ${resolution.height}"), onPressed: () => setResolution(resolution, widget.inferenceProvider)), + ] + ); + } ), - ), + ], ); } } diff --git a/pubspec.lock b/pubspec.lock index 48e2608b..62b26819 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -302,6 +302,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.0.1" + event_bus: + dependency: transitive + description: + name: event_bus + sha256: "1a55e97923769c286d295240048fc180e7b0768902c3c2e869fe059aafa15304" + url: "https://pub.dev" + source: hosted + version: "2.0.1" fake_async: dependency: transitive description: @@ -713,6 +721,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.0.4" + mqtt_client: + dependency: "direct main" + description: + name: mqtt_client + sha256: "85fa7e9aad03fbd6a54fcaf46127579f3e049b4834557a06e49aea7869b04c94" + url: "https://pub.dev" + source: hosted + version: "10.8.0" msix: dependency: "direct dev" description: diff --git a/pubspec.yaml b/pubspec.yaml index 57821a1c..0b054f95 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -74,6 +74,7 @@ dependencies: docx_to_text: ^1.0.1 dart_pdf_reader: ^2.1.0 rxdart: ^0.28.0 + mqtt_client: ^10.8.0 dev_dependencies: flutter_test: @@ -181,7 +182,7 @@ msix_config: display_name: OpenVINO Test Drive publisher_display_name: Intel identity_name: intel.openvino.testdrive - msix_version: 25.1.0.3 + msix_version: 25.1.0.4 trim_logo: false output_name: OpenVINO Test Drive 25.1.0 capabilities: "internetClient" From 8cac4427ec9e53439d7836d04cb4bf55a10796b1 Mon Sep 17 00:00:00 2001 From: Ronald Hecker Date: Wed, 14 May 2025 15:07:24 +0200 Subject: [PATCH 09/12] Make sure the npu is enabled when generating a manifest on import --- lib/pages/import/widgets/directory_import.dart | 2 +- pubspec.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pages/import/widgets/directory_import.dart b/lib/pages/import/widgets/directory_import.dart index df013eb3..64f85cb2 100644 --- a/lib/pages/import/widgets/directory_import.dart +++ b/lib/pages/import/widgets/directory_import.dart @@ -173,7 +173,7 @@ class _ModelImportPropertiesFormState extends State { author: "Unknown", collection: "", description: "Try out $name", - npuEnabled: false, + npuEnabled: true, contextWindow: 0, optimizationPrecision: "", ); diff --git a/pubspec.yaml b/pubspec.yaml index 0b054f95..337aafda 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -182,9 +182,9 @@ msix_config: display_name: OpenVINO Test Drive publisher_display_name: Intel identity_name: intel.openvino.testdrive - msix_version: 25.1.0.4 + msix_version: 25.1.0.5 trim_logo: false - output_name: OpenVINO Test Drive 25.1.0 + output_name: OpenVINO Test Drive 25.1.0.5 capabilities: "internetClient" logo_path: "images/logo_250.png" app_installer: #<-- app installer configuration From 9762f2e27eb74cb4d093966e983a519016593d83 Mon Sep 17 00:00:00 2001 From: Ronald Hecker Date: Wed, 14 May 2025 16:49:45 +0200 Subject: [PATCH 10/12] fix input devices for linux --- openvino_bindings/src/utils/input_devices.h | 22 ++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/openvino_bindings/src/utils/input_devices.h b/openvino_bindings/src/utils/input_devices.h index 2ab2ff7e..5996fd65 100644 --- a/openvino_bindings/src/utils/input_devices.h +++ b/openvino_bindings/src/utils/input_devices.h @@ -15,16 +15,6 @@ #include "status.h" #include "errors.h" - -#if _WIN32 -#include -#include -#include -#include -#include -#pragma comment(lib, "mfplat.lib") -#pragma comment(lib, "mf.lib") - class Resolution { public: int width; @@ -49,6 +39,16 @@ class Camera { }; +#if _WIN32 +#include +#include +#include +#include +#include +#pragma comment(lib, "mfplat.lib") +#pragma comment(lib, "mf.lib") + + std::vector list_camera_devices() { HRESULT hr = MFStartup(MF_VERSION); if (FAILED(hr)) { @@ -140,7 +140,7 @@ std::vector list_camera_devices() { #include std::vector list_camera_devices() { - std::map cameras = {}; + std::vector cameras = {}; for (int i = 0; i < 10; ++i) { // Check up to 10 devices std::string devName = "/dev/video" + std::to_string(i); From 2f630ea0bbdbac98db2e8022acb6b4e7d303bede Mon Sep 17 00:00:00 2001 From: Ronald Hecker Date: Thu, 15 May 2025 13:08:22 +0200 Subject: [PATCH 11/12] Handle empty resolution list for camera device --- lib/pages/computer_vision/live_inference.dart | 12 ++++++++---- lib/pages/computer_vision/widgets/camera_view.dart | 9 ++++++--- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/lib/pages/computer_vision/live_inference.dart b/lib/pages/computer_vision/live_inference.dart index b01a3db7..72ec031a 100644 --- a/lib/pages/computer_vision/live_inference.dart +++ b/lib/pages/computer_vision/live_inference.dart @@ -84,6 +84,13 @@ class _LiveInferenceState extends State { }); } + void closeCamera() { + setState(() { + mode = LiveInferenceMode.image; + cameraDevice = null; + }); + } + @override Widget build(BuildContext context) { final theme = FluentTheme.of(context); @@ -148,10 +155,7 @@ class _LiveInferenceState extends State { }, items: [ MenuFlyoutItem(text: const Text("None"), onPressed: () { - setState(() { - mode = LiveInferenceMode.image; - cameraDevice = null; - }); + closeCamera(); }), for (final device in devices) diff --git a/lib/pages/computer_vision/widgets/camera_view.dart b/lib/pages/computer_vision/widgets/camera_view.dart index e4b4c3c2..e187a751 100644 --- a/lib/pages/computer_vision/widgets/camera_view.dart +++ b/lib/pages/computer_vision/widgets/camera_view.dart @@ -167,12 +167,12 @@ class CameraOptions extends StatefulWidget { class _CameraOptionsState extends State { - late Resolution resolution; + Resolution? resolution; @override void initState() { super.initState(); - resolution = widget.device.resolutions.first; + resolution = widget.device.resolutions.firstOrNull; } void setResolution(Resolution res, ImageInferenceProvider inferenceProvider) { @@ -197,13 +197,16 @@ class _CameraOptionsState extends State { ), Builder( builder: (context) { + if (widget.device.resolutions.isEmpty) { + return Container(); + } return DropDownButton( buttonBuilder: (context, callback) { return NoOutlineButton( onPressed: callback, child: Row( children: [ - Text("Resolution: ${resolution.width} x ${resolution.height}"), + Text("Resolution: ${resolution!.width} x ${resolution!.height}"), const Padding( padding: EdgeInsets.only(left: 8), child: Icon(FluentIcons.chevron_down, size: 12), From e0136d1c2778ac49b202fbc87b3e4b1044810c69 Mon Sep 17 00:00:00 2001 From: Ronald Hecker Date: Thu, 15 May 2025 15:27:02 +0200 Subject: [PATCH 12/12] Fix build for macos and linux, hopefully --- lib/color.dart | 10 ++--- lib/interop/generated_bindings.dart | 31 +-------------- .../widgets/automation_options.dart | 6 +++ lib/pages/import/widgets/model_card.dart | 2 +- .../transcription/widgets/paragraph.dart | 2 +- lib/theme_fluent.dart | 2 +- lib/widgets/canvas/canvas_painter.dart | 2 +- lib/widgets/elevation.dart | 6 +-- macos/Runner.xcodeproj/project.pbxproj | 38 +++++++++---------- macos/Scripts/libraries_tools.sh | 20 +++++----- .../third_party/openvino/linux.BUILD | 3 +- .../third_party/openvino/mac.BUILD | 3 +- pubspec.yaml | 4 ++ 13 files changed, 52 insertions(+), 77 deletions(-) diff --git a/lib/color.dart b/lib/color.dart index b7cd90ca..1e47cc75 100644 --- a/lib/color.dart +++ b/lib/color.dart @@ -15,17 +15,17 @@ extension HexColor on Color { /// Prefixes a hash sign if [leadingHashSign] is set to `true` (default is `true`). String toHex({bool leadingHashSign = true}) => '${leadingHashSign ? '#' : ''}' - '${alpha.toRadixString(16).padLeft(2, '0')}' - '${red.toRadixString(16).padLeft(2, '0')}' - '${green.toRadixString(16).padLeft(2, '0')}' - '${blue.toRadixString(16).padLeft(2, '0')}'; + '${a.toInt().toRadixString(16).padLeft(2, '0')}' + '${r.toInt().toRadixString(16).padLeft(2, '0')}' + '${g.toInt().toRadixString(16).padLeft(2, '0')}' + '${b.toInt().toRadixString(16).padLeft(2, '0')}'; } //float luminance = (0.299f*color.r() + 0.587f*color.g() + 0.114f*color.b()); // Color foregroundColorByLuminance(Color color) { - double luminance = 0.299 * color.red + 0.587 * color.green + 0.144 * color.blue; + double luminance = 0.299 * color.r + 0.587 * color.g + 0.144 * color.b; if (luminance < 128) { return Colors.white; } else { diff --git a/lib/interop/generated_bindings.dart b/lib/interop/generated_bindings.dart index 56aa7525..a266006b 100644 --- a/lib/interop/generated_bindings.dart +++ b/lib/interop/generated_bindings.dart @@ -859,36 +859,7 @@ typedef wint_t = ffi.UnsignedShort; typedef Dartwint_t = int; typedef wctype_t = ffi.UnsignedShort; typedef Dartwctype_t = int; -typedef __time32_t = ffi.Long; -typedef Dart__time32_t = int; -typedef __time64_t = ffi.LongLong; -typedef Dart__time64_t = int; - -final class __crt_locale_data extends ffi.Opaque {} - -final class __crt_multibyte_data extends ffi.Opaque {} - -final class __crt_locale_pointers extends ffi.Struct { - external ffi.Pointer<__crt_locale_data> locinfo; - - external ffi.Pointer<__crt_multibyte_data> mbcinfo; -} - -typedef _locale_t = ffi.Pointer<__crt_locale_pointers>; - -final class _Mbstatet extends ffi.Struct { - @ffi.UnsignedLong() - external int _Wchar; - - @ffi.UnsignedShort() - external int _Byte; - - @ffi.UnsignedShort() - external int _State; -} - -typedef mbstate_t = _Mbstatet; -typedef time_t = __time64_t; +typedef time_t = ffi.LongLong; typedef rsize_t = ffi.Size; typedef Dartrsize_t = int; diff --git a/lib/pages/computer_vision/widgets/automation_options.dart b/lib/pages/computer_vision/widgets/automation_options.dart index 89b20f1c..bddfc9f1 100644 --- a/lib/pages/computer_vision/widgets/automation_options.dart +++ b/lib/pages/computer_vision/widgets/automation_options.dart @@ -1,3 +1,9 @@ +/* + * Copyright (c) 2024 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ + import 'dart:convert'; import 'package:fluent_ui/fluent_ui.dart'; diff --git a/lib/pages/import/widgets/model_card.dart b/lib/pages/import/widgets/model_card.dart index 2e7acb5e..e81bc979 100644 --- a/lib/pages/import/widgets/model_card.dart +++ b/lib/pages/import/widgets/model_card.dart @@ -33,7 +33,7 @@ class ModelCard extends StatelessWidget { decoration: BoxDecoration( borderRadius: const BorderRadius.all(Radius.circular(4)), border: Border.all( - color: checked ? theme.accentColor.withOpacity(0.5) : theme.cardColor, + color: checked ? theme.accentColor.withValues(alpha: 0.5) : theme.cardColor, width: 1.0 ) ), diff --git a/lib/pages/transcription/widgets/paragraph.dart b/lib/pages/transcription/widgets/paragraph.dart index cc274ced..3c9d2d9a 100644 --- a/lib/pages/transcription/widgets/paragraph.dart +++ b/lib/pages/transcription/widgets/paragraph.dart @@ -80,7 +80,7 @@ class _ParagraphState extends State { ), Container( decoration: BoxDecoration( - color: hover ? subtleTextColor.of(theme).withOpacity(0.3) : null, + color: hover ? subtleTextColor.of(theme).withValues(alpha: 0.3) : null, borderRadius: const BorderRadius.all(Radius.circular(4)), ), padding: const EdgeInsets.symmetric(horizontal: 4, vertical: 2), diff --git a/lib/theme_fluent.dart b/lib/theme_fluent.dart index 0d6645c3..54a309a1 100644 --- a/lib/theme_fluent.dart +++ b/lib/theme_fluent.dart @@ -60,7 +60,7 @@ class AppTheme extends ChangeNotifier { WindowEffect.solid, WindowEffect.acrylic, ].contains(effect) - ? FluentTheme.of(context).micaBackgroundColor.withOpacity(0.05) + ? FluentTheme.of(context).micaBackgroundColor.withValues(alpha: 0.05) : Colors.transparent, dark: FluentTheme.of(context).brightness.isDark, ); diff --git a/lib/widgets/canvas/canvas_painter.dart b/lib/widgets/canvas/canvas_painter.dart index f69ca378..2332923b 100644 --- a/lib/widgets/canvas/canvas_painter.dart +++ b/lib/widgets/canvas/canvas_painter.dart @@ -46,7 +46,7 @@ class CanvasPainter extends CustomPainter { ..style = PaintingStyle.stroke; Paint transparent = Paint() - ..color = Color.fromARGB(102, firstLabelColor.red, firstLabelColor.green, firstLabelColor.blue); + ..color = Color.fromARGB(102, firstLabelColor.r.toInt(), firstLabelColor.g.toInt(), firstLabelColor.b.toInt()); if (annotation.shape is Rectangle) { drawRectangle(canvas, size, paint, transparent, annotation); diff --git a/lib/widgets/elevation.dart b/lib/widgets/elevation.dart index b48b452c..2bedd322 100644 --- a/lib/widgets/elevation.dart +++ b/lib/widgets/elevation.dart @@ -24,12 +24,12 @@ class Elevation extends StatelessWidget { shape: shape, shadows: [ BoxShadow( - color: (shadowColor ?? theme.shadowColor).withOpacity(0.13), + color: (shadowColor ?? theme.shadowColor).withValues(alpha: 0.13), blurRadius: 0.9 * elevation, offset: Offset(0, 0.4 * elevation), ), BoxShadow( - color: (shadowColor ?? theme.shadowColor).withOpacity(0.11), + color: (shadowColor ?? theme.shadowColor).withValues(alpha: 0.11), blurRadius: 0.225 * elevation, offset: Offset(0, 0.085 * elevation), ), @@ -38,4 +38,4 @@ class Elevation extends StatelessWidget { child: child, ); } -} \ No newline at end of file +} diff --git a/macos/Runner.xcodeproj/project.pbxproj b/macos/Runner.xcodeproj/project.pbxproj index 03bff430..02f68200 100644 --- a/macos/Runner.xcodeproj/project.pbxproj +++ b/macos/Runner.xcodeproj/project.pbxproj @@ -53,15 +53,15 @@ /* End PBXContainerItemProxy section */ /* Begin PBXFileReference section */ - 0C42C7522CE386520079F72B /* libopenvino_c.2500.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libopenvino_c.2500.dylib; path = ../bindings/libopenvino_c.2500.dylib; sourceTree = SOURCE_ROOT; }; - 0C42C7532CE386520079F72B /* libopenvino_genai.2500.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libopenvino_genai.2500.dylib; path = ../bindings/libopenvino_genai.2500.dylib; sourceTree = SOURCE_ROOT; }; - 0C42C7542CE386520079F72B /* libopenvino_ir_frontend.2500.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libopenvino_ir_frontend.2500.dylib; path = ../bindings/libopenvino_ir_frontend.2500.dylib; sourceTree = SOURCE_ROOT; }; - 0C42C7552CE386520079F72B /* libopenvino_onnx_frontend.2500.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libopenvino_onnx_frontend.2500.dylib; path = ../bindings/libopenvino_onnx_frontend.2500.dylib; sourceTree = SOURCE_ROOT; }; - 0C42C7562CE386520079F72B /* libopenvino_paddle_frontend.2500.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libopenvino_paddle_frontend.2500.dylib; path = ../bindings/libopenvino_paddle_frontend.2500.dylib; sourceTree = SOURCE_ROOT; }; - 0C42C7572CE386520079F72B /* libopenvino_pytorch_frontend.2500.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libopenvino_pytorch_frontend.2500.dylib; path = ../bindings/libopenvino_pytorch_frontend.2500.dylib; sourceTree = SOURCE_ROOT; }; - 0C42C7582CE386520079F72B /* libopenvino_tensorflow_frontend.2500.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libopenvino_tensorflow_frontend.2500.dylib; path = ../bindings/libopenvino_tensorflow_frontend.2500.dylib; sourceTree = SOURCE_ROOT; }; - 0C42C7592CE386520079F72B /* libopenvino_tensorflow_lite_frontend.2500.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libopenvino_tensorflow_lite_frontend.2500.dylib; path = ../bindings/libopenvino_tensorflow_lite_frontend.2500.dylib; sourceTree = SOURCE_ROOT; }; - 0C42C75A2CE386520079F72B /* libopenvino.2500.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libopenvino.2500.dylib; path = ../bindings/libopenvino.2500.dylib; sourceTree = SOURCE_ROOT; }; + 0C42C7522CE386520079F72B /* libopenvino_c.2510.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libopenvino_c.2510.dylib; path = ../bindings/libopenvino_c.2510.dylib; sourceTree = SOURCE_ROOT; }; + 0C42C7532CE386520079F72B /* libopenvino_genai.2510.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libopenvino_genai.2510.dylib; path = ../bindings/libopenvino_genai.2510.dylib; sourceTree = SOURCE_ROOT; }; + 0C42C7542CE386520079F72B /* libopenvino_ir_frontend.2510.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libopenvino_ir_frontend.2510.dylib; path = ../bindings/libopenvino_ir_frontend.2510.dylib; sourceTree = SOURCE_ROOT; }; + 0C42C7552CE386520079F72B /* libopenvino_onnx_frontend.2510.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libopenvino_onnx_frontend.2510.dylib; path = ../bindings/libopenvino_onnx_frontend.2510.dylib; sourceTree = SOURCE_ROOT; }; + 0C42C7562CE386520079F72B /* libopenvino_paddle_frontend.2510.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libopenvino_paddle_frontend.2510.dylib; path = ../bindings/libopenvino_paddle_frontend.2510.dylib; sourceTree = SOURCE_ROOT; }; + 0C42C7572CE386520079F72B /* libopenvino_pytorch_frontend.2510.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libopenvino_pytorch_frontend.2510.dylib; path = ../bindings/libopenvino_pytorch_frontend.2510.dylib; sourceTree = SOURCE_ROOT; }; + 0C42C7582CE386520079F72B /* libopenvino_tensorflow_frontend.2510.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libopenvino_tensorflow_frontend.2510.dylib; path = ../bindings/libopenvino_tensorflow_frontend.2510.dylib; sourceTree = SOURCE_ROOT; }; + 0C42C7592CE386520079F72B /* libopenvino_tensorflow_lite_frontend.2510.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libopenvino_tensorflow_lite_frontend.2510.dylib; path = ../bindings/libopenvino_tensorflow_lite_frontend.2510.dylib; sourceTree = SOURCE_ROOT; }; + 0C42C75A2CE386520079F72B /* libopenvino.2510.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libopenvino.2510.dylib; path = ../bindings/libopenvino.2510.dylib; sourceTree = SOURCE_ROOT; }; 0C4E1F672CECC22800124339 /* libavcodec.60.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libavcodec.60.dylib; path = ../bindings/libavcodec.60.dylib; sourceTree = SOURCE_ROOT; }; 0C4E1F682CECC22800124339 /* libavdevice.60.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libavdevice.60.dylib; path = ../bindings/libavdevice.60.dylib; sourceTree = SOURCE_ROOT; }; 0C4E1F692CECC22800124339 /* libavformat.60.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libavformat.60.dylib; path = ../bindings/libavformat.60.dylib; sourceTree = SOURCE_ROOT; }; @@ -81,7 +81,6 @@ 0C5D476C2C6F397A00307B37 /* libopencv_highgui.410.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libopencv_highgui.410.dylib; path = ../bindings/libopencv_highgui.410.dylib; sourceTree = ""; }; 0C5D476D2C6F397A00307B37 /* libopencv_video.410.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libopencv_video.410.dylib; path = ../bindings/libopencv_video.410.dylib; sourceTree = ""; }; 0C5D47862C6F3A9B00307B37 /* libopenvino_tokenizers.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libopenvino_tokenizers.dylib; path = ../bindings/libopenvino_tokenizers.dylib; sourceTree = ""; }; - 0C5D478B2C6F3A9B00307B37 /* libcore_tokenizers.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libcore_tokenizers.dylib; path = ../bindings/libcore_tokenizers.dylib; sourceTree = ""; }; 0C5D47A32C6F3B7000307B37 /* libtbb.12.dylib */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.dylib"; name = libtbb.12.dylib; path = ../bindings/libtbb.12.dylib; sourceTree = ""; }; 0C5D47A82C6F59A200307B37 /* libopenvino_auto_batch_plugin.so */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.bundle"; name = libopenvino_auto_batch_plugin.so; path = ../bindings/libopenvino_auto_batch_plugin.so; sourceTree = ""; }; 0C5D47A92C6F59A200307B37 /* libopenvino_arm_cpu_plugin.so */ = {isa = PBXFileReference; lastKnownFileType = "compiled.mach-o.bundle"; name = libopenvino_arm_cpu_plugin.so; path = ../bindings/libopenvino_arm_cpu_plugin.so; sourceTree = ""; }; @@ -221,7 +220,6 @@ 0C5D47A82C6F59A200307B37 /* libopenvino_auto_batch_plugin.so */, 0C5D47AB2C6F59A200307B37 /* libopenvino_auto_plugin.so */, 0C5D47AA2C6F59A200307B37 /* libopenvino_hetero_plugin.so */, - 0C5D478B2C6F3A9B00307B37 /* libcore_tokenizers.dylib */, 0C5D47862C6F3A9B00307B37 /* libopenvino_tokenizers.dylib */, 0C5D47602C6F382800307B37 /* libopencv_calib3d.410.dylib */, 0C5D47682C6F397900307B37 /* libopencv_core.410.dylib */, @@ -234,15 +232,15 @@ 0C5D476D2C6F397A00307B37 /* libopencv_video.410.dylib */, 0C5D476B2C6F397A00307B37 /* libopencv_videoio.410.dylib */, 0C5D47A32C6F3B7000307B37 /* libtbb.12.dylib */, - 0C42C7522CE386520079F72B /* libopenvino_c.2500.dylib */, - 0C42C7532CE386520079F72B /* libopenvino_genai.2500.dylib */, - 0C42C7542CE386520079F72B /* libopenvino_ir_frontend.2500.dylib */, - 0C42C7552CE386520079F72B /* libopenvino_onnx_frontend.2500.dylib */, - 0C42C7562CE386520079F72B /* libopenvino_paddle_frontend.2500.dylib */, - 0C42C7572CE386520079F72B /* libopenvino_pytorch_frontend.2500.dylib */, - 0C42C7582CE386520079F72B /* libopenvino_tensorflow_frontend.2500.dylib */, - 0C42C7592CE386520079F72B /* libopenvino_tensorflow_lite_frontend.2500.dylib */, - 0C42C75A2CE386520079F72B /* libopenvino.2500.dylib */, + 0C42C7522CE386520079F72B /* libopenvino_c.2510.dylib */, + 0C42C7532CE386520079F72B /* libopenvino_genai.2510.dylib */, + 0C42C7542CE386520079F72B /* libopenvino_ir_frontend.2510.dylib */, + 0C42C7552CE386520079F72B /* libopenvino_onnx_frontend.2510.dylib */, + 0C42C7562CE386520079F72B /* libopenvino_paddle_frontend.2510.dylib */, + 0C42C7572CE386520079F72B /* libopenvino_pytorch_frontend.2510.dylib */, + 0C42C7582CE386520079F72B /* libopenvino_tensorflow_frontend.2510.dylib */, + 0C42C7592CE386520079F72B /* libopenvino_tensorflow_lite_frontend.2510.dylib */, + 0C42C75A2CE386520079F72B /* libopenvino.2510.dylib */, 0C5D47642C6F397900307B37 /* libopencv_ximgproc.410.dylib */, 0C5D473B2C6F357C00307B37 /* libblend2d.dylib */, 0C5D47372C6F2F9500307B37 /* libmacos_bindings.dylib */, diff --git a/macos/Scripts/libraries_tools.sh b/macos/Scripts/libraries_tools.sh index 043b005e..64465034 100755 --- a/macos/Scripts/libraries_tools.sh +++ b/macos/Scripts/libraries_tools.sh @@ -8,20 +8,20 @@ BUNDLE_FRAMEWORK_LIBS=( "libavutil.58.dylib" "libavdevice.60.dylib" "libavformat.60.dylib" - "libopenvino_paddle_frontend.2500.dylib" + "libopenvino_paddle_frontend.2510.dylib" "libopenvino_hetero_plugin.so" - "libopenvino.2500.dylib" - "libopenvino_onnx_frontend.2500.dylib" + "libopenvino.2510.dylib" + "libopenvino_onnx_frontend.2510.dylib" "libopenvino_auto_batch_plugin.so" "libopenvino_auto_plugin.so" "libblend2d.dylib" "libopencv_core.410.dylib" - "libopenvino_genai.2500.dylib" + "libopenvino_genai.2510.dylib" "libopenvino_arm_cpu_plugin.so" "libopencv_videoio.410.dylib" "libopencv_features2d.410.dylib" - "libopenvino_pytorch_frontend.2500.dylib" - "libopenvino_tensorflow_lite_frontend.2500.dylib" + "libopenvino_pytorch_frontend.2510.dylib" + "libopenvino_tensorflow_lite_frontend.2510.dylib" "libopencv_calib3d.410.dylib" "libopencv_flann.410.dylib" "libopencv_highgui.410.dylib" @@ -31,12 +31,11 @@ BUNDLE_FRAMEWORK_LIBS=( "libopencv_video.410.dylib" "libopencv_ximgproc.410.dylib" "libmacos_bindings.dylib" - "libopenvino_tensorflow_frontend.2500.dylib" + "libopenvino_tensorflow_frontend.2510.dylib" "libtbb.12.dylib" "libopencv_imgcodecs.410.dylib" - "libcore_tokenizers.dylib" - "libopenvino_c.2500.dylib" - "libopenvino_ir_frontend.2500.dylib" + "libopenvino_c.2510.dylib" + "libopenvino_ir_frontend.2510.dylib" ) BUNDLE_FRAMEWORK_LIBS_TO_SIGN=( "libavutil.58.dylib" @@ -62,7 +61,6 @@ BUNDLE_FRAMEWORK_LIBS_TO_SIGN=( "libmacos_bindings.dylib" "libtbb.12.dylib" "libopencv_imgcodecs.410.dylib" - "libcore_tokenizers.dylib" ) Bundle (){ diff --git a/openvino_bindings/third_party/openvino/linux.BUILD b/openvino_bindings/third_party/openvino/linux.BUILD index 1da21883..83434677 100644 --- a/openvino_bindings/third_party/openvino/linux.BUILD +++ b/openvino_bindings/third_party/openvino/linux.BUILD @@ -2,13 +2,12 @@ package( default_visibility = ["//visibility:public"], ) -OPENVINO_VERSION = "2500" +OPENVINO_VERSION = "2510" filegroup( name = "shared_objects", srcs = [ "3rdparty/tbb/lib/libtbb.so.12", - "lib/intel64/libcore_tokenizers.so", "lib/intel64/libopenvino.so." + OPENVINO_VERSION, "lib/intel64/libopenvino_c.so", "lib/intel64/libopenvino_c.so." + OPENVINO_VERSION, diff --git a/openvino_bindings/third_party/openvino/mac.BUILD b/openvino_bindings/third_party/openvino/mac.BUILD index 0f52b164..90b49a43 100644 --- a/openvino_bindings/third_party/openvino/mac.BUILD +++ b/openvino_bindings/third_party/openvino/mac.BUILD @@ -1,4 +1,4 @@ -OPENVINO_VERSION = "2500" +OPENVINO_VERSION = "2510" package( default_visibility = ["//visibility:public"], @@ -36,7 +36,6 @@ cc_library( ]), srcs = [ "lib/arm64/Release/libopenvino_tokenizers.dylib", - "lib/arm64/Release/libcore_tokenizers.dylib", "lib/arm64/Release/libopenvino." + OPENVINO_VERSION +".dylib", "lib/arm64/Release/libopenvino_c." + OPENVINO_VERSION + ".dylib", "lib/arm64/Release/libopenvino_genai." + OPENVINO_VERSION + ".dylib", diff --git a/pubspec.yaml b/pubspec.yaml index 337aafda..9992bbc1 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -164,8 +164,12 @@ ffigen: exclude: - '_.*' globals: + exclude: + - '_(.*)' + typedefs: exclude: - '_.*' + - mbstate_t type-map: native-types: 'char':