diff --git a/examples/README.md b/examples/README.md index e964c52..e96dc6d 100644 --- a/examples/README.md +++ b/examples/README.md @@ -12,6 +12,7 @@ This repository features a collection of real-world applications and walkthrough | [YOLO .Net ONNX Detection C#](https://www.nuget.org/packages/Yolov8.Net) | C# .Net | [Samuel Stainback](https://github.com/sstainba) | | [YOLOv8 on NVIDIA Jetson(TensorRT and DeepStream)](https://wiki.seeedstudio.com/YOLOv8-DeepStream-TRT-Jetson/) | Python | [Lakshantha](https://github.com/lakshanthad) | | [YOLOv8 ONNXRuntime Python](./YOLOv8-ONNXRuntime) | Python/ONNXRuntime | [Semih Demirel](https://github.com/semihhdemirel) | +| [YOLOv8-ONNXRuntime-CPP](./YOLOv8-ONNXRuntime-CPP) | C++/ONNXRuntime | [DennisJcy](https://github.com/DennisJcy) | ### How to Contribute diff --git a/examples/YOLOv8-ONNXRuntime-CPP/README.md b/examples/YOLOv8-ONNXRuntime-CPP/README.md new file mode 100644 index 0000000..c498664 --- /dev/null +++ b/examples/YOLOv8-ONNXRuntime-CPP/README.md @@ -0,0 +1,54 @@ +# YOLOv8 OnnxRuntime C++ + +This example demonstrates how to perform inference using YOLOv8 in C++ with ONNX Runtime and OpenCV's API. + +We recommend using Visual Studio to build the project. + +## Benefits + +- Friendly for deployment in the industrial sector. +- Faster than OpenCV's DNN inference on both CPU and GPU. +- Supports CUDA acceleration. +- Easy to add FP16 inference (using template functions). + +## Exporting YOLOv8 Models + +To export YOLOv8 models, use the following Python script: + +```python +from ultralytics import YOLO + +# Load a YOLOv8 model +model = YOLO("yolov8n.pt") + +# Export the model +model.export(format="onnx", opset=12, simplify=True, dynamic=False, imgsz=640) +``` + +## Dependencies + +| Dependency | Version | +| ----------------------- | -------- | +| Onnxruntime-win-x64-gpu | >=1.14.1 | +| OpenCV | >=4.0.0 | +| C++ | >=17 | + +Note: The dependency on C++17 is due to the usage of the C++17 filesystem feature. + +## Usage + +```c++ +// CPU inference +DCSP_INIT_PARAM params{ model_path, YOLO_ORIGIN_V8, {imgsz_w, imgsz_h}, class_num, 0.1, 0.5, false}; +// GPU inference +DCSP_INIT_PARAM params{ model_path, YOLO_ORIGIN_V8, {imgsz_w, imgsz_h}, class_num, 0.1, 0.5, true}; + +// Load your image +cv::Mat img = cv::imread(img_path); + +char* ret = p1->CreateSession(params); + +ret = p->RunSession(img, res); +``` + +This repository should also work for YOLOv5, which needs a permute operator for the output of the YOLOv5 model, but this has not been implemented yet. diff --git a/examples/YOLOv8-ONNXRuntime-CPP/inference.cpp b/examples/YOLOv8-ONNXRuntime-CPP/inference.cpp new file mode 100644 index 0000000..5af395d --- /dev/null +++ b/examples/YOLOv8-ONNXRuntime-CPP/inference.cpp @@ -0,0 +1,271 @@ +#include "inference.h" +#include + +#define benchmark +#define ELOG + +DCSP_CORE::DCSP_CORE() +{ + +} + + +DCSP_CORE::~DCSP_CORE() +{ + delete session; +} + + +template +char* BlobFromImage(cv::Mat& iImg, T& iBlob) +{ + int channels = iImg.channels(); + int imgHeight = iImg.rows; + int imgWidth = iImg.cols; + + for (int c = 0; c < channels; c++) + { + for (int h = 0; h < imgHeight; h++) + { + for (int w = 0; w < imgWidth; w++) + { + iBlob[c * imgWidth * imgHeight + h * imgWidth + w] = (std::remove_pointer::type)((iImg.at(h, w)[c]) / 255.0f); + } + } + } + return RET_OK; +} + + +char* PostProcess(cv::Mat& iImg, std::vector iImgSize, cv::Mat& oImg) +{ + cv::Mat img = iImg.clone(); + cv::resize(iImg, oImg, cv::Size(iImgSize.at(0), iImgSize.at(1))); + if (img.channels() == 1) + { + cv::cvtColor(oImg, oImg, cv::COLOR_GRAY2BGR); + } + cv::cvtColor(oImg, oImg, cv::COLOR_BGR2RGB); + return RET_OK; +} + + +char* DCSP_CORE::CreateSession(DCSP_INIT_PARAM &iParams) +{ + char* Ret = RET_OK; + std::regex pattern("[\u4e00-\u9fa5]"); + bool result = std::regex_search(iParams.ModelPath, pattern); + if (result) + { + Ret = "[DCSP_ONNX]:model path error.change your model path without chinese characters."; + std::cout << Ret << std::endl; + return Ret; + } + try + { + rectConfidenceThreshold = iParams.RectConfidenceThreshold; + iouThreshold = iParams.iouThreshold; + imgSize = iParams.imgSize; + modelType = iParams.ModelType; + env = Ort::Env(ORT_LOGGING_LEVEL_WARNING, "Yolo"); + Ort::SessionOptions sessionOption; + if (iParams.CudaEnable) + { + cudaEnable = iParams.CudaEnable; + OrtCUDAProviderOptions cudaOption; + cudaOption.device_id = 0; + sessionOption.AppendExecutionProvider_CUDA(cudaOption); + //OrtOpenVINOProviderOptions ovOption; + //sessionOption.AppendExecutionProvider_OpenVINO(ovOption); + } + sessionOption.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); + sessionOption.SetIntraOpNumThreads(iParams.IntraOpNumThreads); + sessionOption.SetLogSeverityLevel(iParams.LogSeverityLevel); + int ModelPathSize = MultiByteToWideChar(CP_UTF8, 0, iParams.ModelPath.c_str(), static_cast(iParams.ModelPath.length()), nullptr, 0); + wchar_t* wide_cstr = new wchar_t[ModelPathSize + 1]; + MultiByteToWideChar(CP_UTF8, 0, iParams.ModelPath.c_str(), static_cast(iParams.ModelPath.length()), wide_cstr, ModelPathSize); + wide_cstr[ModelPathSize] = L'\0'; + const wchar_t* modelPath = wide_cstr; + session = new Ort::Session(env, modelPath, sessionOption); + Ort::AllocatorWithDefaultOptions allocator; + size_t inputNodesNum = session->GetInputCount(); + for (size_t i = 0; i < inputNodesNum; i++) + { + Ort::AllocatedStringPtr input_node_name = session->GetInputNameAllocated(i, allocator); + char* temp_buf = new char[50]; + strcpy(temp_buf, input_node_name.get()); + inputNodeNames.push_back(temp_buf); + } + + size_t OutputNodesNum = session->GetOutputCount(); + for (size_t i = 0; i < OutputNodesNum; i++) + { + Ort::AllocatedStringPtr output_node_name = session->GetOutputNameAllocated(i, allocator); + char* temp_buf = new char[10]; + strcpy(temp_buf, output_node_name.get()); + outputNodeNames.push_back(temp_buf); + } + options = Ort::RunOptions{ nullptr }; + WarmUpSession(); + //std::cout << OrtGetApiBase()->GetVersionString() << std::endl;; + Ret = RET_OK; + return Ret; + } + catch (const std::exception& e) + { + const char* str1 = "[DCSP_ONNX]:"; + const char* str2 = e.what(); + std::string result = std::string(str1) + std::string(str2); + char* merged = new char[result.length() + 1]; + std::strcpy(merged, result.c_str()); + std::cout << merged << std::endl; + delete[] merged; + //return merged; + return "[DCSP_ONNX]:Create session failed."; + } + +} + + +char* DCSP_CORE::RunSession(cv::Mat &iImg, std::vector& oResult) +{ +#ifdef benchmark + clock_t starttime_1 = clock(); +#endif // benchmark + + char* Ret = RET_OK; + cv::Mat processedImg; + PostProcess(iImg, imgSize, processedImg); + if (modelType < 4) + { + float* blob = new float[processedImg.total() * 3]; + BlobFromImage(processedImg, blob); + std::vector inputNodeDims = { 1,3,imgSize.at(0),imgSize.at(1) }; + TensorProcess(starttime_1, iImg, blob, inputNodeDims, oResult); + } + + return Ret; +} + + +template +char* DCSP_CORE::TensorProcess(clock_t& starttime_1, cv::Mat& iImg, N& blob, std::vector& inputNodeDims, std::vector& oResult) +{ + Ort::Value inputTensor = Ort::Value::CreateTensor::type>(Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU), blob, 3 * imgSize.at(0) * imgSize.at(1), inputNodeDims.data(), inputNodeDims.size()); +#ifdef benchmark + clock_t starttime_2 = clock(); +#endif // benchmark + auto outputTensor = session->Run(options, inputNodeNames.data(), &inputTensor, 1, outputNodeNames.data(), outputNodeNames.size()); +#ifdef benchmark + clock_t starttime_3 = clock(); +#endif // benchmark + Ort::TypeInfo typeInfo = outputTensor.front().GetTypeInfo(); + auto tensor_info = typeInfo.GetTensorTypeAndShapeInfo(); + std::vectoroutputNodeDims = tensor_info.GetShape(); + std::remove_pointer::type* output = outputTensor.front().GetTensorMutableData::type>(); + delete blob; + switch (modelType) + { + case 1: + { + int strideNum = outputNodeDims[2]; + int signalResultNum = outputNodeDims[1]; + std::vector class_ids; + std::vector confidences; + std::vector boxes; + cv::Mat rowData(signalResultNum, strideNum, CV_32F, output); + rowData = rowData.t(); + + float* data = (float*)rowData.data; + + float x_factor = iImg.cols / 640.; + float y_factor = iImg.rows / 640.; + for (int i = 0; i < strideNum; ++i) + { + float* classesScores = data + 4; + cv::Mat scores(1, classesNum, CV_32FC1, classesScores); + cv::Point class_id; + double maxClassScore; + cv::minMaxLoc(scores, 0, &maxClassScore, 0, &class_id); + if (maxClassScore > rectConfidenceThreshold) + { + confidences.push_back(maxClassScore); + class_ids.push_back(class_id.x); + + float x = data[0]; + float y = data[1]; + float w = data[2]; + float h = data[3]; + + int left = int((x - 0.5 * w) * x_factor); + int top = int((y - 0.5 * h) * y_factor); + + int width = int(w * x_factor); + int height = int(h * y_factor); + + boxes.push_back(cv::Rect(left, top, width, height)); + } + data += signalResultNum; + } + + std::vector nmsResult; + cv::dnn::NMSBoxes(boxes, confidences, rectConfidenceThreshold, iouThreshold, nmsResult); + for (int i = 0; i < nmsResult.size(); ++i) + { + int idx = nmsResult[i]; + DCSP_RESULT result; + result.classId = class_ids[idx]; + result.confidence = confidences[idx]; + result.box = boxes[idx]; + oResult.push_back(result); + } + + +#ifdef benchmark + clock_t starttime_4 = clock(); + double pre_process_time = (double)(starttime_2 - starttime_1) / CLOCKS_PER_SEC * 1000; + double process_time = (double)(starttime_3 - starttime_2) / CLOCKS_PER_SEC * 1000; + double post_process_time = (double)(starttime_4 - starttime_3) / CLOCKS_PER_SEC * 1000; + if (cudaEnable) + { + std::cout << "[DCSP_ONNX(CUDA)]: " << pre_process_time << "ms pre-process, " << process_time << "ms inference, " << post_process_time << "ms post-process." << std::endl; + } + else + { + std::cout << "[DCSP_ONNX(CPU)]: " << pre_process_time << "ms pre-process, " << process_time << "ms inference, " << post_process_time << "ms post-process." << std::endl; + } +#endif // benchmark + + break; + } + } + char* Ret = RET_OK; + return Ret; +} + + +char* DCSP_CORE::WarmUpSession() +{ + clock_t starttime_1 = clock(); + char* Ret = RET_OK; + cv::Mat iImg = cv::Mat(cv::Size(imgSize.at(0), imgSize.at(1)), CV_8UC3); + cv::Mat processedImg; + PostProcess(iImg, imgSize, processedImg); + if (modelType < 4) + { + float* blob = new float[iImg.total() * 3]; + BlobFromImage(processedImg, blob); + std::vector YOLO_input_node_dims = { 1,3,imgSize.at(0),imgSize.at(1) }; + Ort::Value input_tensor = Ort::Value::CreateTensor(Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU), blob, 3 * imgSize.at(0) * imgSize.at(1), YOLO_input_node_dims.data(), YOLO_input_node_dims.size()); + auto output_tensors = session->Run(options, inputNodeNames.data(), &input_tensor, 1, outputNodeNames.data(), outputNodeNames.size()); + delete[] blob; + clock_t starttime_4 = clock(); + double post_process_time = (double)(starttime_4 - starttime_1) / CLOCKS_PER_SEC * 1000; + if (cudaEnable) + { + std::cout << "[DCSP_ONNX(CUDA)]: " << "Cuda warm-up cost " << post_process_time << " ms. " << std::endl; + } + } + + return Ret; +} diff --git a/examples/YOLOv8-ONNXRuntime-CPP/inference.h b/examples/YOLOv8-ONNXRuntime-CPP/inference.h new file mode 100644 index 0000000..d00fecd --- /dev/null +++ b/examples/YOLOv8-ONNXRuntime-CPP/inference.h @@ -0,0 +1,83 @@ +#pragma once + +#define _CRT_SECURE_NO_WARNINGS +#define RET_OK nullptr + +#include +#include +#include +#include "io.h" +#include "direct.h" +#include "opencv.hpp" +#include +#include "onnxruntime_cxx_api.h" + + +enum MODEL_TYPE +{ + //FLOAT32 MODEL + YOLO_ORIGIN_V5 = 0, + YOLO_ORIGIN_V8 = 1,//only support v8 detector currently + YOLO_POSE_V8 = 2, + YOLO_CLS_V8 = 3 +}; + + +typedef struct _DCSP_INIT_PARAM +{ + std::string ModelPath; + MODEL_TYPE ModelType = YOLO_ORIGIN_V8; + std::vector imgSize={640, 640}; + + int classesNum=80; + float RectConfidenceThreshold = 0.6; + float iouThreshold = 0.5; + bool CudaEnable = false; + int LogSeverityLevel = 3; + int IntraOpNumThreads = 1; +}DCSP_INIT_PARAM; + + +typedef struct _DCSP_RESULT +{ + int classId; + float confidence; + cv::Rect box; +}DCSP_RESULT; + + +class DCSP_CORE +{ +public: + DCSP_CORE(); + ~DCSP_CORE(); + +public: + char* CreateSession(DCSP_INIT_PARAM &iParams); + + + char* RunSession(cv::Mat &iImg, std::vector& oResult); + + + char* WarmUpSession(); + + + template + char* TensorProcess(clock_t& starttime_1, cv::Mat& iImg, N& blob, std::vector& inputNodeDims, std::vector& oResult); + + +private: + Ort::Env env; + Ort::Session* session; + bool cudaEnable; + Ort::RunOptions options; + std::vector inputNodeNames; + std::vector outputNodeNames; + + + int classesNum; + MODEL_TYPE modelType; + std::vector imgSize; + float rectConfidenceThreshold; + float iouThreshold; +}; diff --git a/examples/YOLOv8-ONNXRuntime-CPP/main.cpp b/examples/YOLOv8-ONNXRuntime-CPP/main.cpp new file mode 100644 index 0000000..f13d782 --- /dev/null +++ b/examples/YOLOv8-ONNXRuntime-CPP/main.cpp @@ -0,0 +1,44 @@ +#include +#include +#include "inference.h" +#include + + + +void file_iterator(DCSP_CORE*& p) +{ + std::filesystem::path img_path = R"(E:\project\Project_C++\DCPS_ONNX\TEST_ORIGIN)"; + int k = 0; + for (auto& i : std::filesystem::directory_iterator(img_path)) + { + if (i.path().extension() == ".jpg") + { + std::string img_path = i.path().string(); + //std::cout << img_path << std::endl; + cv::Mat img = cv::imread(img_path); + std::vector res; + char* ret = p->RunSession(img, res); + for (int i = 0; i < res.size(); i++) + { + cv::rectangle(img, res.at(i).box, cv::Scalar(125, 123, 0), 3); + } + + k++; + cv::imshow("TEST_ORIGIN", img); + cv::waitKey(0); + cv::destroyAllWindows(); + //cv::imwrite("E:\\output\\" + std::to_string(k) + ".png", img); + } + } +} + + + +int main() +{ + DCSP_CORE* p1 = new DCSP_CORE; + std::string model_path = "yolov8n.onnx"; + DCSP_INIT_PARAM params{ model_path, YOLO_ORIGIN_V8, {640, 640}, 80, 0.1, 0.5, false }; + char* ret = p1->CreateSession(params); + file_iterator(p1); +}