Integration of OpenVINO and CoreML (Apple Silicon)

ConsoleDemo/Program.cs

@@ -64,7 +64,7 @@ static void RunDemo(ModelType modelType, ModelVersion modelVersion, ImageType im
     using var yolo = new Yolo(new YoloOptions()
     {
         OnnxModel = modelPath,
-        Cuda = cuda,
+        HwAccelerator = cuda ? HwAcceleratorType.Cuda : HwAcceleratorType.None,
         PrimeGpu = primeGpu,
         ModelType = modelType,
     });
@@ -151,7 +151,7 @@ static void ObjectDetectionOnVideo()
     {
         OnnxModel = SharedConfig.GetTestModelV8(ModelType.ObjectDetection),
         ModelType = ModelType.ObjectDetection,
-        Cuda = true
+        HwAccelerator = HwAcceleratorType.Cuda
     });
 
     int currentLineCursor = 0;

README.md

@@ -55,6 +55,31 @@ ONNX runtime's [current compatibility with specific versions](https://onnxruntim
 
 4. Super-duper-important! In order for Windows to pick up the changes in your Environment Variables, make sure to close all open programs before you continue with whatever you were doing ;)
 
+5. Add `Microsoft.ML.OnnxRuntime @ 1.19.2` and `Microsoft.ML.OnnxRuntime.Gpu @ 1.19.2` to your project.
+
+# Install OpenVINO (optional)
+YoloDotNet with CPU/GPU/NPU-acceleration provided by OpenVINO requires OpenVINO Toolkit installed.
+
+ONNX runtime's [current compatibility with specific versions](https://onnxruntime.ai/docs/execution-providers/OpenVINO-ExecutionProvider.html#requirements).
+
+1. Download [OpenVINO Toolkit](https://github.com/openvinotoolkit/openvino/releases) ([Download 2024.3.0 archive](https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.3/)) and extract inside `C:\Program Files (x86)\Intel\openvino_2024.3.0`
+
+2. Update your system PATH-variable to include OpenVINO [instructions](https://www.intel.com/content/www/us/en/support/articles/000033440/software/development-software.html)
+
+3. Super-duper-important! In order for Windows to pick up the changes in your Environment Variables, make sure to close all open programs before you continue with whatever you were doing ;)
+
+4. Download latest release of [Microsoft.ML.OnnxRuntime.OpenVino](https://github.com/intel/onnxruntime/releases) shared library build and reference the package in your project. [1.19.0 with ONNXRuntime 5.4](https://github.com/intel/onnxruntime/releases/download/v5.4/Microsoft.ML.OnnxRuntime.OpenVino.1.19.0.zip)
+    - preferred way is to install it via local NuGet source via the provided .nupkg from the release page linked above
+    - Add **ONLY** `Microsoft.ML.OnnxRuntime.OpenVino @ 1.19.0` (without `Microsoft.ML.OnnxRuntime`) to your project.
+
+5. Set `HwAccelerator = HwAcceleratorType.OpenVino` in YoloDotNet's options
+
+# Use CoreML / Apple Silicon acceleration (optional)
+
+1. Add `Microsoft.ML.OnnxRuntime @ 1.19.2` to your project.
+
+2. Set `HwAccelerator = HwAcceleratorType.CoreML` in YoloDotNet's options
+
 # Export Yolo models to ONNX
 All models must be exported to ONNX format. [How to export to ONNX format](https://docs.ultralytics.com/modes/export/#usage-examples).\
 The ONNX-models included in this repo are from Ultralytics s-series (small). https://docs.ultralytics.com/models.
@@ -84,7 +109,7 @@ using var yolo = new Yolo(new YoloOptions
 {
     OnnxModel = @"path\to\model.onnx",      // Your Yolo model in onnx format
     ModelType = ModelType.ObjectDetection,  // Set your model type
-    Cuda = false,                           // Use CPU or CUDA for GPU accelerated inference. Default = true
+    HwAccelerator = HwAcceleratorType.None, // Use CPU, CUDA or OpenVINO for GPU accelerated inference. Default = CUDA
     GpuId = 0                               // Select Gpu by id. Default = 0
     PrimeGpu = false,                       // Pre-allocate GPU before first inference. Default = false
 });
@@ -117,7 +142,7 @@ using System.Threading.Tasks;
 using var yolo = new Yolo(new YoloOptions()
 {
     OnnxModel = @"path\to\model.onnx",
-    Cuda = true,
+    HwAccelerator = HwAcceleratorType.Cuda, // Use CPU, CUDA or OpenVINO for GPU accelerated inference. Default = CUDA
     PrimeGpu = true,
     ModelType = ModelType.ObjectDetection
 });
@@ -162,7 +187,7 @@ using var yolo = new Yolo(new YoloOptions
 {
     OnnxModel = @"path\to\model.onnx",      // Your Yolov8 or Yolov10 model in onnx format
     ModelType = ModelType.ObjectDetection,  // Set your model type
-    Cuda = false,                           // Use CPU or CUDA for GPU accelerated inference. Default = true
+    HwAccelerator = HwAcceleratorType.None, // Use CPU, CUDA or OpenVINO for GPU accelerated inference. Default = CUDA
     GpuId = 0                               // Select Gpu by id. Default = 0
     PrimeGpu = false,                       // Pre-allocate GPU before first. Default = false
 });
@@ -274,6 +299,10 @@ for (var i = 0; i < labels.Length; i++)
 
 # References & Acknowledgements
 
+https://github.com/microsoft/onnxruntime
+
+https://github.com/intel/onnxruntime
+
 https://github.com/ultralytics/ultralytics
 
 https://github.com/sstainba/Yolov8.Net

YoloDotNet/Core/ModuleFactory.cs

@@ -92,7 +92,7 @@ public static IModule CreateModule(YoloOptions options)
         /// <returns>An initialized YoloCore instance.</returns>
         private static YoloCore InitializeYoloCore(YoloOptions options)
         {
-            var yoloCore = new YoloCore(options.OnnxModel, options.Cuda, options.PrimeGpu, options.GpuId);
+            var yoloCore = new YoloCore(options.OnnxModel, options.HwAccelerator);
             yoloCore.InitializeYolo(options);
             return yoloCore;
         }

YoloDotNet/Core/YoloCore.cs

@@ -1,13 +1,13 @@
-namespace YoloDotNet.Core
+using System;
+
+namespace YoloDotNet.Core
 {
     /// <summary>
     /// Initializes a new instance of the Yolo core class.
     /// </summary>
     /// <param name="onnxModel">The path to the ONNX model file to use.</param>
-    /// <param name="useCuda">Indicates whether to use CUDA for GPU acceleration.</param>
-    /// <param name="allocateGpuMemory">Indicates whether to allocate GPU memory.</param>
-    /// <param name="gpuId">The GPU device ID to use when CUDA is enabled.</param>
-    public class YoloCore(string onnxModel, bool useCuda, bool allocateGpuMemory, int gpuId) : IDisposable
+    /// <param name="hwAccelerator">Indicates which HW device to use for acceleration.</param>
+    public class YoloCore(string onnxModel, HwAcceleratorType hwAccelerator) : IDisposable
     {
         public event EventHandler VideoStatusEvent = delegate { };
         public event EventHandler VideoProgressEvent = delegate { };
@@ -34,9 +34,17 @@ public class YoloCore(string onnxModel, bool useCuda, bool allocateGpuMemory, in
         /// <param name="modelType">The type of the model to be initialized.</param>
         public void InitializeYolo(YoloOptions yoloOptions)
         {
-            _session = useCuda
-                ? new InferenceSession(onnxModel, SessionOptions.MakeSessionOptionWithCudaProvider(gpuId))
-                : new InferenceSession(onnxModel);
+            var sessionOptions = new SessionOptions();
+
+            // apply HW accelerator
+            if (hwAccelerator == HwAcceleratorType.OpenVino)
+                sessionOptions.AppendExecutionProvider_OpenVINO(yoloOptions.OpenVinoDeviceId);
+            else if (hwAccelerator == HwAcceleratorType.Cuda)
+                sessionOptions.AppendExecutionProvider_CUDA(yoloOptions.GpuId);
+            else if (hwAccelerator == HwAcceleratorType.CoreML)
+                sessionOptions.AppendExecutionProvider_CoreML();
+
+            _session = new InferenceSession(onnxModel, sessionOptions);
 
             _runOptions = new RunOptions();
             _ortIoBinding = _session.CreateIoBinding();
@@ -54,7 +62,7 @@ public void InitializeYolo(YoloOptions yoloOptions)
 
             _imageInfo = new SKImageInfo(OnnxModel.Input.Width, OnnxModel.Input.Height, SKColorType.Rgb888x, SKAlphaType.Opaque);
 
-            if (useCuda && allocateGpuMemory)
+            if (hwAccelerator == HwAcceleratorType.Cuda && yoloOptions.PrimeGpu)
                 _session.AllocateGpuMemory(_ortIoBinding, _runOptions, customSizeFloatPool, _imageInfo);
         }
 
@@ -110,7 +118,7 @@ public Dictionary<int, List<T>> RunVideo<T>(
         {
             var output = new Dictionary<int, List<T>>();
 
-            using var _videoHandler = new VideoHandler.VideoHandler(options, useCuda);
+            using var _videoHandler = new VideoHandler.VideoHandler(options, hwAccelerator == HwAcceleratorType.Cuda);
 
             _videoHandler.ProgressEvent += (sender, e) => VideoProgressEvent?.Invoke(sender, e);
             _videoHandler.VideoCompleteEvent += (sender, e) => VideoCompleteEvent?.Invoke(sender, e);

YoloDotNet/Enums/HwAcceleratorType.cs

@@ -0,0 +1,13 @@
+namespace YoloDotNet.Enums
+{
+    /// <summary>
+    /// Hardware acceletator type
+    /// </summary>
+    public enum HwAcceleratorType
+    {
+        None = 0,
+        Cuda = 1,
+        OpenVino = 2,
+        CoreML = 3
+    }
+}

YoloDotNet/Models/YoloOptions.cs

@@ -16,9 +16,9 @@ public class YoloOptions
         public ModelType ModelType { get; set; }
 
         /// <summary>
-        /// Gets or sets a value indicating whether to use CUDA for GPU acceleration (default is true).
+        /// Choose hardware accelerator (default is HwAcceleratorType.Cuda)
         /// </summary>
-        public bool Cuda { get; set; } = true;
+        public HwAcceleratorType HwAccelerator { get; set; } = HwAcceleratorType.Cuda;
 
         /// <summary>
         /// Gets or sets a value indicating whether to prime the GPU before inference (default is false).
@@ -29,5 +29,10 @@ public class YoloOptions
         /// Gets or sets the ID of the GPU to use (default is 0).
         /// </summary>
         public int GpuId { get; set; } = 0;
+
+        /// <summary>
+        /// Gets or sets the ID of the OpenVINO device to use (default is NPU) (CPU, GPU or NPU).
+        /// </summary>
+        public string OpenVinoDeviceId = "NPU";
     }
 }

YoloDotNet/YoloDotNet.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net9.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
@@ -40,15 +40,8 @@ Need more control? An optional pixel confidence parameter has been added for seg
   </PropertyGroup>
 
   <ItemGroup>
-    <None Include="D:\Documents\Yolodotnet Icon\icon.png">
-      <Pack>True</Pack>
-      <PackagePath>\</PackagePath>
-    </None>
-  </ItemGroup>
-
-  <ItemGroup>
-    <PackageReference Include="Microsoft.ML.OnnxRuntime.Gpu" Version="1.19.2" />
-    <PackageReference Include="SkiaSharp" Version="2.88.8" />
+    <PackageReference Include="Microsoft.ML.OnnxRuntime.Managed" Version="1.20.2" />
+    <PackageReference Include="SkiaSharp" Version="3.116.1" />
   </ItemGroup>
 
 </Project>

test/YoloDotNet.Benchmarks/ClassificationTests/SimpleClassificationTests.cs

@@ -34,19 +34,19 @@ public void GlobalSetup()
             // Yolov8
             options.OnnxModel = _model8;
 
-            options.Cuda = false;
+            options.HwAccelerator = HwAcceleratorType.None;
             _cpuYolov8 = new Yolo(options);
 
-            options.Cuda = true;
+            options.HwAccelerator = HwAcceleratorType.Cuda;
             _gpuYolov8 = new Yolo(options);
 
             // Yolov11
             options.OnnxModel = _model11;
 
-            options.Cuda = false;
+            options.HwAccelerator = HwAcceleratorType.None;
             _cpuYolov11 = new Yolo(options);
 
-            options.Cuda = true;
+            options.HwAccelerator = HwAcceleratorType.Cuda;
             _gpuYolov11 = new Yolo(options);
         }
 

test/YoloDotNet.Benchmarks/ImageExtensionTests/ClassificationImageDrawTests.cs

@@ -23,7 +23,7 @@ public void GlobalSetup()
             {
                 OnnxModel = _model,
                 ModelType = ModelType.Classification,
-                Cuda = false
+                HwAccelerator = HwAcceleratorType.None
             };
 
             _cpuYolo = new Yolo(options);

test/YoloDotNet.Benchmarks/ImageExtensionTests/NormalizePixelsToTensorTests.cs

@@ -29,7 +29,7 @@ public void GlobalSetup()
             {
                 OnnxModel = _model,
                 ModelType = ModelType.ObjectDetection,
-                Cuda = false
+                HwAccelerator = HwAcceleratorType.None
             };
 
             _cpuYolo = new Yolo(options);

test/YoloDotNet.Benchmarks/ImageExtensionTests/ObjectDetectionImageDrawTests.cs

@@ -23,7 +23,7 @@ public void GlobalSetup()
             {
                 OnnxModel = _model,
                 ModelType = ModelType.ObjectDetection,
-                Cuda = false
+                HwAccelerator = HwAcceleratorType.None
             };
 
             _cpuYolo = new Yolo(options);

test/YoloDotNet.Benchmarks/ImageExtensionTests/OrientedBoundingBoxImageDrawTests.cs

@@ -23,7 +23,7 @@ public void GlobalSetup()
             {
                 OnnxModel = _model,
                 ModelType = ModelType.ObbDetection,
-                Cuda = false
+                HwAccelerator = HwAcceleratorType.None
             };
 
             _cpuYolo = new Yolo(options);

test/YoloDotNet.Benchmarks/ImageExtensionTests/PoseEstimationImageDrawTests.cs

@@ -23,7 +23,7 @@ public void GlobalSetup()
             {
                 OnnxModel = _model,
                 ModelType = ModelType.PoseEstimation,
-                Cuda = false
+                HwAccelerator = HwAcceleratorType.None
             };
 
             _cpuYolo = new Yolo(options);

test/YoloDotNet.Benchmarks/ImageExtensionTests/SegmentationImageDrawTests.cs

@@ -23,7 +23,7 @@ public void GlobalSetup()
             {
                 OnnxModel = _model,
                 ModelType = ModelType.Segmentation,
-                Cuda = false
+                HwAccelerator = HwAcceleratorType.None
             };
 
             _cpuYolo = new Yolo(options);

test/YoloDotNet.Benchmarks/ObjectDetectionTests/ResizeSourceObjectDetectionTests.cs

@@ -25,12 +25,12 @@ public void GlobalSetup()
             {
                 OnnxModel = _model,
                 ModelType = ModelType.ObjectDetection,
-                Cuda = false
+                HwAccelerator = HwAcceleratorType.None
             };
 
             _cpuYolo = new Yolo(options);
 
-            options.Cuda = true;
+            options.HwAccelerator = HwAcceleratorType.Cuda;
             _cudaYolo = new Yolo(options);
 
             _originalSizeimage = SKImage.FromEncodedData(_originalSizeimagePath);

test/YoloDotNet.Benchmarks/ObjectDetectionTests/SimpleObjectDetectionTests.cs

@@ -31,41 +31,41 @@ public void GlobalSetup()
             var options = new YoloOptions
             {
                 ModelType = ModelType.ObjectDetection,
-                Cuda = false
+                HwAccelerator = HwAcceleratorType.None
             };
 
             // Yolov8
             options.OnnxModel = _modelV8;
             _cpuYolov8 = new Yolo(options);
 
-            options.Cuda = true;
+            options.HwAccelerator = HwAcceleratorType.Cuda;
             _gpuYolov8 = new Yolo(options);
 
             // Yolov9
             options.OnnxModel = _modelV9;
-            options.Cuda = false;
+            options.HwAccelerator = HwAcceleratorType.None;
 
             _cpuYolov9 = new Yolo(options);
 
-            options.Cuda = true;
+            options.HwAccelerator = HwAcceleratorType.Cuda;
             _gpuYolov9 = new Yolo(options);
 
             // Yolov10
             options.OnnxModel = _modelV10;
-            options.Cuda = false;
+            options.HwAccelerator = HwAcceleratorType.None;
 
             _cpuYolov10 = new Yolo(options);
 
-            options.Cuda = true;
+            options.HwAccelerator = HwAcceleratorType.Cuda;
             _gpuYolov10 = new Yolo(options);
 
             // Yolov11
             options.OnnxModel = _modelV11;
-            options.Cuda = false;
+            options.HwAccelerator = HwAcceleratorType.None;
 
             _cpuYolov11 = new Yolo(options);
 
-            options.Cuda = true;
+            options.HwAccelerator = HwAcceleratorType.Cuda;
             _gpuYolov11 = new Yolo(options);
         }
 

test/YoloDotNet.Benchmarks/OrientedBoundingBoxTests/SimpleOrientedBoundingBoxTests.cs

@@ -34,19 +34,19 @@ public void GlobalSetup()
             // Yolov8
             options.OnnxModel = _model8;
 
-            options.Cuda = false;
+            options.HwAccelerator = HwAcceleratorType.None;
             _cpuYolov8 = new Yolo(options);
 
-            options.Cuda = true;
+            options.HwAccelerator = HwAcceleratorType.Cuda;
             _gpuYolov8 = new Yolo(options);
 
             // Yolov11
             options.OnnxModel = _model11;
 
-            options.Cuda = false;
+            options.HwAccelerator = HwAcceleratorType.None;
             _cpuYolov11 = new Yolo(options);
 
-            options.Cuda = true;
+            options.HwAccelerator = HwAcceleratorType.Cuda;
             _gpuYolov11 = new Yolo(options);
         }