From b5d1af42d8e34e4019a2c6c031e5fd71dde811d5 Mon Sep 17 00:00:00 2001 From: Onuralp SEZER Date: Sun, 13 Aug 2023 18:34:39 +0300 Subject: [PATCH] Update YOLOv8-ONNXRuntime-CPP example with GPU inference (#4328) Signed-off-by: Onuralp SEZER Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Glenn Jocher --- examples/README.md | 20 +- .../YOLOv8-ONNXRuntime-CPP/CMakeLists.txt | 69 +-- examples/YOLOv8-ONNXRuntime-CPP/README.md | 19 +- examples/YOLOv8-ONNXRuntime-CPP/inference.cpp | 471 +++++++++--------- examples/YOLOv8-ONNXRuntime-CPP/inference.h | 98 ++-- examples/YOLOv8-ONNXRuntime-CPP/main.cpp | 71 +-- 6 files changed, 375 insertions(+), 373 deletions(-) diff --git a/examples/README.md b/examples/README.md index 120c04b..f740b7c 100644 --- a/examples/README.md +++ b/examples/README.md @@ -4,16 +4,16 @@ This repository features a collection of real-world applications and walkthrough ### Ultralytics YOLO Example Applications -| Title | Format | Contributor | -| -------------------------------------------------------------------------------------------------------------- | ------------------ | --------------------------------------------------- | -| [YOLO ONNX Detection Inference with C++](./YOLOv8-CPP-Inference) | C++/ONNX | [Justas Bartnykas](https://github.com/JustasBart) | -| [YOLO OpenCV ONNX Detection Python](./YOLOv8-OpenCV-ONNX-Python) | OpenCV/Python/ONNX | [Farid Inawan](https://github.com/frdteknikelektro) | -| [YOLOv8 .NET ONNX ImageSharp](https://github.com/dme-compunet/YOLOv8) | C#/ONNX/ImageSharp | [Compunet](https://github.com/dme-compunet) | -| [YOLO .Net ONNX Detection C#](https://www.nuget.org/packages/Yolov8.Net) | C# .Net | [Samuel Stainback](https://github.com/sstainba) | -| [YOLOv8 on NVIDIA Jetson(TensorRT and DeepStream)](https://wiki.seeedstudio.com/YOLOv8-DeepStream-TRT-Jetson/) | Python | [Lakshantha](https://github.com/lakshanthad) | -| [YOLOv8 ONNXRuntime Python](./YOLOv8-ONNXRuntime) | Python/ONNXRuntime | [Semih Demirel](https://github.com/semihhdemirel) | -| [YOLOv8-ONNXRuntime-CPP](./YOLOv8-ONNXRuntime-CPP) | C++/ONNXRuntime | [DennisJcy](https://github.com/DennisJcy) | -| [RTDETR ONNXRuntime C#](https://github.com/Kayzwer/yolo-cs/blob/master/RTDETR.cs) | C#/ONNX | [Kayzwer](https://github.com/Kayzwer) | +| Title | Format | Contributor | +| -------------------------------------------------------------------------------------------------------------- | ------------------ | ----------------------------------------------------------------------------------------- | +| [YOLO ONNX Detection Inference with C++](./YOLOv8-CPP-Inference) | C++/ONNX | [Justas Bartnykas](https://github.com/JustasBart) | +| [YOLO OpenCV ONNX Detection Python](./YOLOv8-OpenCV-ONNX-Python) | OpenCV/Python/ONNX | [Farid Inawan](https://github.com/frdteknikelektro) | +| [YOLOv8 .NET ONNX ImageSharp](https://github.com/dme-compunet/YOLOv8) | C#/ONNX/ImageSharp | [Compunet](https://github.com/dme-compunet) | +| [YOLO .Net ONNX Detection C#](https://www.nuget.org/packages/Yolov8.Net) | C# .Net | [Samuel Stainback](https://github.com/sstainba) | +| [YOLOv8 on NVIDIA Jetson(TensorRT and DeepStream)](https://wiki.seeedstudio.com/YOLOv8-DeepStream-TRT-Jetson/) | Python | [Lakshantha](https://github.com/lakshanthad) | +| [YOLOv8 ONNXRuntime Python](./YOLOv8-ONNXRuntime) | Python/ONNXRuntime | [Semih Demirel](https://github.com/semihhdemirel) | +| [YOLOv8-ONNXRuntime-CPP](./YOLOv8-ONNXRuntime-CPP) | C++/ONNXRuntime | [DennisJcy](https://github.com/DennisJcy), [Onuralp Sezer](https://github.com/onuralpszr) | +| [RTDETR ONNXRuntime C#](https://github.com/Kayzwer/yolo-cs/blob/master/RTDETR.cs) | C#/ONNX | [Kayzwer](https://github.com/Kayzwer) | ### How to Contribute diff --git a/examples/YOLOv8-ONNXRuntime-CPP/CMakeLists.txt b/examples/YOLOv8-ONNXRuntime-CPP/CMakeLists.txt index 41a4148..494a6f1 100644 --- a/examples/YOLOv8-ONNXRuntime-CPP/CMakeLists.txt +++ b/examples/YOLOv8-ONNXRuntime-CPP/CMakeLists.txt @@ -17,58 +17,69 @@ include_directories(${OpenCV_INCLUDE_DIRS}) # -------------- Compile CUDA for FP16 inference if needed ------------------# -find_package(CUDA REQUIRED) -include_directories(${CUDA_INCLUDE_DIRS}) - +option(USE_CUDA "Enable CUDA support" ON) +if (USE_CUDA) + find_package(CUDA REQUIRED) + include_directories(${CUDA_INCLUDE_DIRS}) + add_definitions(-DUSE_CUDA) +endif () # ONNXRUNTIME # Set ONNXRUNTIME_VERSION set(ONNXRUNTIME_VERSION 1.15.1) -if(WIN32) - # CPU - # set(ONNXRUNTIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime-win-x64-${ONNXRUNTIME_VERSION}") - # GPU - set(ONNXRUNTIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime-win-x64-gpu-${ONNXRUNTIME_VERSION}") -elseif(LINUX) - # CPU - # set(ONNXRUNTIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}") - # GPU - set(ONNXRUNTIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime-linux-x64-gpu-${ONNXRUNTIME_VERSION}") -elseif(APPLE) +if (WIN32) + if (USE_CUDA) + set(ONNXRUNTIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime-win-x64-gpu-${ONNXRUNTIME_VERSION}") + else () + set(ONNXRUNTIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime-win-x64-${ONNXRUNTIME_VERSION}") + endif () +elseif (LINUX) + if (USE_CUDA) + set(ONNXRUNTIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime-linux-x64-gpu-${ONNXRUNTIME_VERSION}") + else () + set(ONNXRUNTIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}") + endif () +elseif (APPLE) set(ONNXRUNTIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime-osx-arm64-${ONNXRUNTIME_VERSION}") # Apple X64 binary # set(ONNXRUNTIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime-osx-x64-${ONNXRUNTIME_VERSION}") # Apple Universal binary # set(ONNXRUNTIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime-osx-universal2-${ONNXRUNTIME_VERSION}") -endif() +endif () include_directories(${PROJECT_NAME} ${ONNXRUNTIME_ROOT}/include) set(PROJECT_SOURCES - main.cpp - inference.h - inference.cpp + main.cpp + inference.h + inference.cpp ) add_executable(${PROJECT_NAME} ${PROJECT_SOURCES}) -if(WIN32) - target_link_libraries(${PROJECT_NAME} ${OpenCV_LIBS} ${ONNXRUNTIME_ROOT}/lib/onnxruntime.lib ${CUDA_LIBRARIES}) -elseif(LINUX) - target_link_libraries(${PROJECT_NAME} ${OpenCV_LIBS} ${ONNXRUNTIME_ROOT}/lib/libonnxruntime.so ${CUDA_LIBRARIES}) -elseif(APPLE) +if (WIN32) + target_link_libraries(${PROJECT_NAME} ${OpenCV_LIBS} ${ONNXRUNTIME_ROOT}/lib/onnxruntime.lib) + if (USE_CUDA) + target_link_libraries(${PROJECT_NAME} ${CUDA_LIBRARIES}) + endif () +elseif (LINUX) + target_link_libraries(${PROJECT_NAME} ${OpenCV_LIBS} ${ONNXRUNTIME_ROOT}/lib/libonnxruntime.so) + if (USE_CUDA) + target_link_libraries(${PROJECT_NAME} ${CUDA_LIBRARIES}) + endif () +elseif (APPLE) target_link_libraries(${PROJECT_NAME} ${OpenCV_LIBS} ${ONNXRUNTIME_ROOT}/lib/libonnxruntime.dylib) -endif() +endif () # For windows system, copy onnxruntime.dll to the same folder of the executable file -if(WIN32) +if (WIN32) add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different - "${ONNXRUNTIME_ROOT}/lib/onnxruntime.dll" - $) -endif() + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${ONNXRUNTIME_ROOT}/lib/onnxruntime.dll" + $) +endif () # Download https://raw.githubusercontent.com/ultralytics/ultralytics/main/ultralytics/cfg/datasets/coco.yaml # and put it in the same folder of the executable file diff --git a/examples/YOLOv8-ONNXRuntime-CPP/README.md b/examples/YOLOv8-ONNXRuntime-CPP/README.md index 87326a4..91fb3bc 100644 --- a/examples/YOLOv8-ONNXRuntime-CPP/README.md +++ b/examples/YOLOv8-ONNXRuntime-CPP/README.md @@ -28,16 +28,23 @@ Alternatively, you can use the following command for exporting the model in the yolo export model=yolov8n.pt opset=12 simplify=True dynamic=False format=onnx imgsz=640,640 ``` +## Download COCO.yaml file + +In order to run example, you also need to download coco.yaml. You can download the file manually from [here](https://raw.githubusercontent.com/ultralytics/ultralytics/main/ultralytics/cfg/datasets/coco.yaml) + ## Dependencies -| Dependency | Version | -| -------------------------------- | -------- | -| Onnxruntime(linux,windows,macos) | >=1.14.1 | -| OpenCV | >=4.0.0 | -| C++ | >=17 | -| Cmake | >=3.5 | +| Dependency | Version | +| -------------------------------- | ------------- | +| Onnxruntime(linux,windows,macos) | >=1.14.1 | +| OpenCV | >=4.0.0 | +| C++ | >=17 | +| Cmake | >=3.5 | +| Cuda (Optional) | >=11.4,\<12.0 | +| cuDNN (Cuda required) | =8 | Note: The dependency on C++17 is due to the usage of the C++17 filesystem feature. +Note (2): Due to ONNX Runtime, we need to use CUDA 11 and cuDNN 8. Keep in mind that this requirement might change in the future. ## Usage diff --git a/examples/YOLOv8-ONNXRuntime-CPP/inference.cpp b/examples/YOLOv8-ONNXRuntime-CPP/inference.cpp index 7e67cd5..6a772c7 100644 --- a/examples/YOLOv8-ONNXRuntime-CPP/inference.cpp +++ b/examples/YOLOv8-ONNXRuntime-CPP/inference.cpp @@ -3,297 +3,280 @@ #define benchmark -DCSP_CORE::DCSP_CORE() -{ +DCSP_CORE::DCSP_CORE() { } -DCSP_CORE::~DCSP_CORE() -{ - delete session; +DCSP_CORE::~DCSP_CORE() { + delete session; } - +#ifdef USE_CUDA namespace Ort { - template<> - struct TypeToTensorType { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16; }; + template<> + struct TypeToTensorType { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16; }; } +#endif template -char* BlobFromImage(cv::Mat& iImg, T& iBlob) -{ - int channels = iImg.channels(); - int imgHeight = iImg.rows; - int imgWidth = iImg.cols; - - for (int c = 0; c < channels; c++) - { - for (int h = 0; h < imgHeight; h++) - { - for (int w = 0; w < imgWidth; w++) - { - iBlob[c * imgWidth * imgHeight + h * imgWidth + w] = typename std::remove_pointer::type((iImg.at(h, w)[c]) / 255.0f); - } - } - } - return RET_OK; +char *BlobFromImage(cv::Mat &iImg, T &iBlob) { + int channels = iImg.channels(); + int imgHeight = iImg.rows; + int imgWidth = iImg.cols; + + for (int c = 0; c < channels; c++) { + for (int h = 0; h < imgHeight; h++) { + for (int w = 0; w < imgWidth; w++) { + iBlob[c * imgWidth * imgHeight + h * imgWidth + w] = typename std::remove_pointer::type( + (iImg.at(h, w)[c]) / 255.0f); + } + } + } + return RET_OK; } -char* PostProcess(cv::Mat& iImg, std::vector iImgSize, cv::Mat& oImg) -{ - cv::Mat img = iImg.clone(); +char *PostProcess(cv::Mat &iImg, std::vector iImgSize, cv::Mat &oImg) { + cv::Mat img = iImg.clone(); cv::resize(iImg, oImg, cv::Size(iImgSize.at(0), iImgSize.at(1))); - if (img.channels() == 1) - { - cv::cvtColor(oImg, oImg, cv::COLOR_GRAY2BGR); - } - cv::cvtColor(oImg, oImg, cv::COLOR_BGR2RGB); - return RET_OK; + if (img.channels() == 1) { + cv::cvtColor(oImg, oImg, cv::COLOR_GRAY2BGR); + } + cv::cvtColor(oImg, oImg, cv::COLOR_BGR2RGB); + return RET_OK; } -char* DCSP_CORE::CreateSession(DCSP_INIT_PARAM &iParams) -{ - char* Ret = RET_OK; - std::regex pattern("[\u4e00-\u9fa5]"); - bool result = std::regex_search(iParams.ModelPath, pattern); - if (result) - { - Ret = "[DCSP_ONNX]:Model path error.Change your model path without chinese characters."; - std::cout << Ret << std::endl; - return Ret; - } - try - { - rectConfidenceThreshold = iParams.RectConfidenceThreshold; - iouThreshold = iParams.iouThreshold; - imgSize = iParams.imgSize; - modelType = iParams.ModelType; - env = Ort::Env(ORT_LOGGING_LEVEL_WARNING, "Yolo"); - Ort::SessionOptions sessionOption; - if (iParams.CudaEnable) - { - cudaEnable = iParams.CudaEnable; - OrtCUDAProviderOptions cudaOption; - cudaOption.device_id = 0; - sessionOption.AppendExecutionProvider_CUDA(cudaOption); - } - sessionOption.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); - sessionOption.SetIntraOpNumThreads(iParams.IntraOpNumThreads); - sessionOption.SetLogSeverityLevel(iParams.LogSeverityLevel); +char *DCSP_CORE::CreateSession(DCSP_INIT_PARAM &iParams) { + char *Ret = RET_OK; + std::regex pattern("[\u4e00-\u9fa5]"); + bool result = std::regex_search(iParams.ModelPath, pattern); + if (result) { + Ret = "[DCSP_ONNX]:Model path error.Change your model path without chinese characters."; + std::cout << Ret << std::endl; + return Ret; + } + try { + rectConfidenceThreshold = iParams.RectConfidenceThreshold; + iouThreshold = iParams.iouThreshold; + imgSize = iParams.imgSize; + modelType = iParams.ModelType; + env = Ort::Env(ORT_LOGGING_LEVEL_WARNING, "Yolo"); + Ort::SessionOptions sessionOption; + if (iParams.CudaEnable) { + cudaEnable = iParams.CudaEnable; + OrtCUDAProviderOptions cudaOption; + cudaOption.device_id = 0; + sessionOption.AppendExecutionProvider_CUDA(cudaOption); + } + sessionOption.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); + sessionOption.SetIntraOpNumThreads(iParams.IntraOpNumThreads); + sessionOption.SetLogSeverityLevel(iParams.LogSeverityLevel); #ifdef _WIN32 - int ModelPathSize = MultiByteToWideChar(CP_UTF8, 0, iParams.ModelPath.c_str(), static_cast(iParams.ModelPath.length()), nullptr, 0); - wchar_t* wide_cstr = new wchar_t[ModelPathSize + 1]; - MultiByteToWideChar(CP_UTF8, 0, iParams.ModelPath.c_str(), static_cast(iParams.ModelPath.length()), wide_cstr, ModelPathSize); - wide_cstr[ModelPathSize] = L'\0'; - const wchar_t* modelPath = wide_cstr; + int ModelPathSize = MultiByteToWideChar(CP_UTF8, 0, iParams.ModelPath.c_str(), static_cast(iParams.ModelPath.length()), nullptr, 0); + wchar_t* wide_cstr = new wchar_t[ModelPathSize + 1]; + MultiByteToWideChar(CP_UTF8, 0, iParams.ModelPath.c_str(), static_cast(iParams.ModelPath.length()), wide_cstr, ModelPathSize); + wide_cstr[ModelPathSize] = L'\0'; + const wchar_t* modelPath = wide_cstr; #else - const char* modelPath = iParams.ModelPath.c_str(); + const char *modelPath = iParams.ModelPath.c_str(); #endif // _WIN32 - session = new Ort::Session(env, modelPath, sessionOption); - Ort::AllocatorWithDefaultOptions allocator; - size_t inputNodesNum = session->GetInputCount(); - for (size_t i = 0; i < inputNodesNum; i++) - { - Ort::AllocatedStringPtr input_node_name = session->GetInputNameAllocated(i, allocator); - char* temp_buf = new char[50]; - strcpy(temp_buf, input_node_name.get()); - inputNodeNames.push_back(temp_buf); - } - size_t OutputNodesNum = session->GetOutputCount(); - for (size_t i = 0; i < OutputNodesNum; i++) - { - Ort::AllocatedStringPtr output_node_name = session->GetOutputNameAllocated(i, allocator); - char* temp_buf = new char[10]; - strcpy(temp_buf, output_node_name.get()); - outputNodeNames.push_back(temp_buf); - } - options = Ort::RunOptions{ nullptr }; - WarmUpSession(); - return RET_OK; - } - catch (const std::exception& e) - { - const char* str1 = "[DCSP_ONNX]:"; - const char* str2 = e.what(); - std::string result = std::string(str1) + std::string(str2); - char* merged = new char[result.length() + 1]; - std::strcpy(merged, result.c_str()); - std::cout << merged << std::endl; - delete[] merged; - return "[DCSP_ONNX]:Create session failed."; - } + session = new Ort::Session(env, modelPath, sessionOption); + Ort::AllocatorWithDefaultOptions allocator; + size_t inputNodesNum = session->GetInputCount(); + for (size_t i = 0; i < inputNodesNum; i++) { + Ort::AllocatedStringPtr input_node_name = session->GetInputNameAllocated(i, allocator); + char *temp_buf = new char[50]; + strcpy(temp_buf, input_node_name.get()); + inputNodeNames.push_back(temp_buf); + } + size_t OutputNodesNum = session->GetOutputCount(); + for (size_t i = 0; i < OutputNodesNum; i++) { + Ort::AllocatedStringPtr output_node_name = session->GetOutputNameAllocated(i, allocator); + char *temp_buf = new char[10]; + strcpy(temp_buf, output_node_name.get()); + outputNodeNames.push_back(temp_buf); + } + options = Ort::RunOptions{nullptr}; + WarmUpSession(); + return RET_OK; + } + catch (const std::exception &e) { + const char *str1 = "[DCSP_ONNX]:"; + const char *str2 = e.what(); + std::string result = std::string(str1) + std::string(str2); + char *merged = new char[result.length() + 1]; + std::strcpy(merged, result.c_str()); + std::cout << merged << std::endl; + delete[] merged; + return "[DCSP_ONNX]:Create session failed."; + } } -char* DCSP_CORE::RunSession(cv::Mat &iImg, std::vector& oResult) -{ +char *DCSP_CORE::RunSession(cv::Mat &iImg, std::vector &oResult) { #ifdef benchmark - clock_t starttime_1 = clock(); + clock_t starttime_1 = clock(); #endif // benchmark - char* Ret = RET_OK; - cv::Mat processedImg; - PostProcess(iImg, imgSize, processedImg); - if (modelType < 4) - { - float* blob = new float[processedImg.total() * 3]; - BlobFromImage(processedImg, blob); - std::vector inputNodeDims = { 1,3,imgSize.at(0),imgSize.at(1) }; - TensorProcess(starttime_1, iImg, blob, inputNodeDims, oResult); - } - else - { - half* blob = new half[processedImg.total() * 3]; - BlobFromImage(processedImg, blob); - std::vector inputNodeDims = { 1,3,imgSize.at(0),imgSize.at(1) }; - TensorProcess(starttime_1, iImg, blob, inputNodeDims, oResult); - } - - return Ret; + char *Ret = RET_OK; + cv::Mat processedImg; + PostProcess(iImg, imgSize, processedImg); + if (modelType < 4) { + float *blob = new float[processedImg.total() * 3]; + BlobFromImage(processedImg, blob); + std::vector inputNodeDims = {1, 3, imgSize.at(0), imgSize.at(1)}; + TensorProcess(starttime_1, iImg, blob, inputNodeDims, oResult); + } else { +#ifdef USE_CUDA + half* blob = new half[processedImg.total() * 3]; + BlobFromImage(processedImg, blob); + std::vector inputNodeDims = { 1,3,imgSize.at(0),imgSize.at(1) }; + TensorProcess(starttime_1, iImg, blob, inputNodeDims, oResult); +#endif + } + + return Ret; } template -char* DCSP_CORE::TensorProcess(clock_t& starttime_1, cv::Mat& iImg, N& blob, std::vector& inputNodeDims, std::vector& oResult) -{ - Ort::Value inputTensor = Ort::Value::CreateTensor::type>(Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU), blob, 3 * imgSize.at(0) * imgSize.at(1), inputNodeDims.data(), inputNodeDims.size()); +char *DCSP_CORE::TensorProcess(clock_t &starttime_1, cv::Mat &iImg, N &blob, std::vector &inputNodeDims, + std::vector &oResult) { + Ort::Value inputTensor = Ort::Value::CreateTensor::type>( + Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU), blob, 3 * imgSize.at(0) * imgSize.at(1), + inputNodeDims.data(), inputNodeDims.size()); #ifdef benchmark - clock_t starttime_2 = clock(); + clock_t starttime_2 = clock(); #endif // benchmark - auto outputTensor = session->Run(options, inputNodeNames.data(), &inputTensor, 1, outputNodeNames.data(), outputNodeNames.size()); + auto outputTensor = session->Run(options, inputNodeNames.data(), &inputTensor, 1, outputNodeNames.data(), + outputNodeNames.size()); #ifdef benchmark - clock_t starttime_3 = clock(); + clock_t starttime_3 = clock(); #endif // benchmark - Ort::TypeInfo typeInfo = outputTensor.front().GetTypeInfo(); - auto tensor_info = typeInfo.GetTensorTypeAndShapeInfo(); - std::vectoroutputNodeDims = tensor_info.GetShape(); + Ort::TypeInfo typeInfo = outputTensor.front().GetTypeInfo(); + auto tensor_info = typeInfo.GetTensorTypeAndShapeInfo(); + std::vector outputNodeDims = tensor_info.GetShape(); auto output = outputTensor.front().GetTensorMutableData::type>(); - delete blob; - switch (modelType) - { - case 1://V8_ORIGIN_FP32 - case 4://V8_ORIGIN_FP16 - { - int strideNum = outputNodeDims[2]; - int signalResultNum = outputNodeDims[1]; - std::vector class_ids; - std::vector confidences; - std::vector boxes; - cv::Mat rowData(signalResultNum, strideNum, CV_32F, output); - rowData = rowData.t(); - - float* data = (float*)rowData.data; - - float x_factor = iImg.cols / 640.; - float y_factor = iImg.rows / 640.; - for (int i = 0; i < strideNum; ++i) - { - float* classesScores = data + 4; - cv::Mat scores(1, this->classes.size(), CV_32FC1, classesScores); - cv::Point class_id; - double maxClassScore; - cv::minMaxLoc(scores, 0, &maxClassScore, 0, &class_id); - if (maxClassScore > rectConfidenceThreshold) - { - confidences.push_back(maxClassScore); - class_ids.push_back(class_id.x); - - float x = data[0]; - float y = data[1]; - float w = data[2]; - float h = data[3]; - - int left = int((x - 0.5 * w) * x_factor); - int top = int((y - 0.5 * h) * y_factor); - - int width = int(w * x_factor); - int height = int(h * y_factor); - - boxes.emplace_back(left, top, width, height); - } - data += signalResultNum; - } - - std::vector nmsResult; - cv::dnn::NMSBoxes(boxes, confidences, rectConfidenceThreshold, iouThreshold, nmsResult); - - for (int i = 0; i < nmsResult.size(); ++i) - { - int idx = nmsResult[i]; - DCSP_RESULT result; - result.classId = class_ids[idx]; - result.confidence = confidences[idx]; - result.box = boxes[idx]; - oResult.push_back(result); - } + delete blob; + switch (modelType) { + case 1://V8_ORIGIN_FP32 + case 4://V8_ORIGIN_FP16 + { + int strideNum = outputNodeDims[2]; + int signalResultNum = outputNodeDims[1]; + std::vector class_ids; + std::vector confidences; + std::vector boxes; + cv::Mat rowData(signalResultNum, strideNum, CV_32F, output); + rowData = rowData.t(); + + float *data = (float *) rowData.data; + + float x_factor = iImg.cols / 640.; + float y_factor = iImg.rows / 640.; + for (int i = 0; i < strideNum; ++i) { + float *classesScores = data + 4; + cv::Mat scores(1, this->classes.size(), CV_32FC1, classesScores); + cv::Point class_id; + double maxClassScore; + cv::minMaxLoc(scores, 0, &maxClassScore, 0, &class_id); + if (maxClassScore > rectConfidenceThreshold) { + confidences.push_back(maxClassScore); + class_ids.push_back(class_id.x); + + float x = data[0]; + float y = data[1]; + float w = data[2]; + float h = data[3]; + + int left = int((x - 0.5 * w) * x_factor); + int top = int((y - 0.5 * h) * y_factor); + + int width = int(w * x_factor); + int height = int(h * y_factor); + + boxes.emplace_back(left, top, width, height); + } + data += signalResultNum; + } + + std::vector nmsResult; + cv::dnn::NMSBoxes(boxes, confidences, rectConfidenceThreshold, iouThreshold, nmsResult); + + for (int i = 0; i < nmsResult.size(); ++i) { + int idx = nmsResult[i]; + DCSP_RESULT result; + result.classId = class_ids[idx]; + result.confidence = confidences[idx]; + result.box = boxes[idx]; + oResult.push_back(result); + } #ifdef benchmark - clock_t starttime_4 = clock(); - double pre_process_time = (double)(starttime_2 - starttime_1) / CLOCKS_PER_SEC * 1000; - double process_time = (double)(starttime_3 - starttime_2) / CLOCKS_PER_SEC * 1000; - double post_process_time = (double)(starttime_4 - starttime_3) / CLOCKS_PER_SEC * 1000; - if (cudaEnable) - { - std::cout << "[DCSP_ONNX(CUDA)]: " << pre_process_time << "ms pre-process, " << process_time << "ms inference, " << post_process_time << "ms post-process." << std::endl; - } - else - { - std::cout << "[DCSP_ONNX(CPU)]: " << pre_process_time << "ms pre-process, " << process_time << "ms inference, " << post_process_time << "ms post-process." << std::endl; - } + clock_t starttime_4 = clock(); + double pre_process_time = (double) (starttime_2 - starttime_1) / CLOCKS_PER_SEC * 1000; + double process_time = (double) (starttime_3 - starttime_2) / CLOCKS_PER_SEC * 1000; + double post_process_time = (double) (starttime_4 - starttime_3) / CLOCKS_PER_SEC * 1000; + if (cudaEnable) { + std::cout << "[DCSP_ONNX(CUDA)]: " << pre_process_time << "ms pre-process, " << process_time + << "ms inference, " << post_process_time << "ms post-process." << std::endl; + } else { + std::cout << "[DCSP_ONNX(CPU)]: " << pre_process_time << "ms pre-process, " << process_time + << "ms inference, " << post_process_time << "ms post-process." << std::endl; + } #endif // benchmark - break; - } - } - return RET_OK; + break; + } + } + return RET_OK; } -char* DCSP_CORE::WarmUpSession() -{ - clock_t starttime_1 = clock(); - cv::Mat iImg = cv::Mat(cv::Size(imgSize.at(0), imgSize.at(1)), CV_8UC3); - cv::Mat processedImg; - PostProcess(iImg, imgSize, processedImg); - if (modelType < 4) - { - float* blob = new float[iImg.total() * 3]; - BlobFromImage(processedImg, blob); - std::vector YOLO_input_node_dims = { 1,3,imgSize.at(0),imgSize.at(1) }; - Ort::Value input_tensor = Ort::Value::CreateTensor(Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU), blob, 3 * imgSize.at(0) * imgSize.at(1), YOLO_input_node_dims.data(), YOLO_input_node_dims.size()); - auto output_tensors = session->Run(options, inputNodeNames.data(), &input_tensor, 1, outputNodeNames.data(), outputNodeNames.size()); - delete[] blob; - clock_t starttime_4 = clock(); - double post_process_time = (double)(starttime_4 - starttime_1) / CLOCKS_PER_SEC * 1000; - if (cudaEnable) - { - std::cout << "[DCSP_ONNX(CUDA)]: " << "Cuda warm-up cost " << post_process_time << " ms. " << std::endl; - } - } - else - { - half* blob = new half[iImg.total() * 3]; - BlobFromImage(processedImg, blob); - std::vector YOLO_input_node_dims = { 1,3,imgSize.at(0),imgSize.at(1) }; - Ort::Value input_tensor = Ort::Value::CreateTensor(Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU), blob, 3 * imgSize.at(0) * imgSize.at(1), YOLO_input_node_dims.data(), YOLO_input_node_dims.size()); - auto output_tensors = session->Run(options, inputNodeNames.data(), &input_tensor, 1, outputNodeNames.data(), outputNodeNames.size()); - delete[] blob; - clock_t starttime_4 = clock(); - double post_process_time = (double)(starttime_4 - starttime_1) / CLOCKS_PER_SEC * 1000; - if (cudaEnable) - { - std::cout << "[DCSP_ONNX(CUDA)]: " << "Cuda warm-up cost " << post_process_time << " ms. " << std::endl; - } - } - return RET_OK; +char *DCSP_CORE::WarmUpSession() { + clock_t starttime_1 = clock(); + cv::Mat iImg = cv::Mat(cv::Size(imgSize.at(0), imgSize.at(1)), CV_8UC3); + cv::Mat processedImg; + PostProcess(iImg, imgSize, processedImg); + if (modelType < 4) { + float *blob = new float[iImg.total() * 3]; + BlobFromImage(processedImg, blob); + std::vector YOLO_input_node_dims = {1, 3, imgSize.at(0), imgSize.at(1)}; + Ort::Value input_tensor = Ort::Value::CreateTensor( + Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU), blob, 3 * imgSize.at(0) * imgSize.at(1), + YOLO_input_node_dims.data(), YOLO_input_node_dims.size()); + auto output_tensors = session->Run(options, inputNodeNames.data(), &input_tensor, 1, outputNodeNames.data(), + outputNodeNames.size()); + delete[] blob; + clock_t starttime_4 = clock(); + double post_process_time = (double) (starttime_4 - starttime_1) / CLOCKS_PER_SEC * 1000; + if (cudaEnable) { + std::cout << "[DCSP_ONNX(CUDA)]: " << "Cuda warm-up cost " << post_process_time << " ms. " << std::endl; + } + } else { +#ifdef USE_CUDA + half* blob = new half[iImg.total() * 3]; + BlobFromImage(processedImg, blob); + std::vector YOLO_input_node_dims = { 1,3,imgSize.at(0),imgSize.at(1) }; + Ort::Value input_tensor = Ort::Value::CreateTensor(Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU), blob, 3 * imgSize.at(0) * imgSize.at(1), YOLO_input_node_dims.data(), YOLO_input_node_dims.size()); + auto output_tensors = session->Run(options, inputNodeNames.data(), &input_tensor, 1, outputNodeNames.data(), outputNodeNames.size()); + delete[] blob; + clock_t starttime_4 = clock(); + double post_process_time = (double)(starttime_4 - starttime_1) / CLOCKS_PER_SEC * 1000; + if (cudaEnable) + { + std::cout << "[DCSP_ONNX(CUDA)]: " << "Cuda warm-up cost " << post_process_time << " ms. " << std::endl; + } +#endif + } + return RET_OK; } diff --git a/examples/YOLOv8-ONNXRuntime-CPP/inference.h b/examples/YOLOv8-ONNXRuntime-CPP/inference.h index a1db199..fe2c5a0 100644 --- a/examples/YOLOv8-ONNXRuntime-CPP/inference.h +++ b/examples/YOLOv8-ONNXRuntime-CPP/inference.h @@ -1,6 +1,6 @@ #pragma once -#define RET_OK nullptr +#define RET_OK nullptr #ifdef _WIN32 #include @@ -13,72 +13,72 @@ #include #include #include "onnxruntime_cxx_api.h" + +#ifdef USE_CUDA #include +#endif -enum MODEL_TYPE -{ - //FLOAT32 MODEL - YOLO_ORIGIN_V5 = 0, - YOLO_ORIGIN_V8 = 1,//only support v8 detector currently - YOLO_POSE_V8 = 2, - YOLO_CLS_V8 = 3, - YOLO_ORIGIN_V8_HALF = 4, - YOLO_POSE_V8_HALF = 5, - YOLO_CLS_V8_HALF = 6 +enum MODEL_TYPE { + //FLOAT32 MODEL + YOLO_ORIGIN_V5 = 0, + YOLO_ORIGIN_V8 = 1,//only support v8 detector currently + YOLO_POSE_V8 = 2, + YOLO_CLS_V8 = 3, + YOLO_ORIGIN_V8_HALF = 4, + YOLO_POSE_V8_HALF = 5, + YOLO_CLS_V8_HALF = 6 }; +typedef struct _DCSP_INIT_PARAM { + std::string ModelPath; + MODEL_TYPE ModelType = YOLO_ORIGIN_V8; + std::vector imgSize = {640, 640}; + float RectConfidenceThreshold = 0.6; + float iouThreshold = 0.5; + bool CudaEnable = false; + int LogSeverityLevel = 3; + int IntraOpNumThreads = 1; +} DCSP_INIT_PARAM; -typedef struct _DCSP_INIT_PARAM -{ - std::string ModelPath; - MODEL_TYPE ModelType = YOLO_ORIGIN_V8; - std::vector imgSize={640, 640}; - float RectConfidenceThreshold = 0.6; - float iouThreshold = 0.5; - bool CudaEnable = false; - int LogSeverityLevel = 3; - int IntraOpNumThreads = 1; -}DCSP_INIT_PARAM; +typedef struct _DCSP_RESULT { + int classId; + float confidence; + cv::Rect box; +} DCSP_RESULT; -typedef struct _DCSP_RESULT -{ - int classId; - float confidence; - cv::Rect box; -}DCSP_RESULT; - -class DCSP_CORE -{ +class DCSP_CORE { public: - DCSP_CORE(); - ~DCSP_CORE(); + DCSP_CORE(); + + ~DCSP_CORE(); public: - char* CreateSession(DCSP_INIT_PARAM &iParams); + char *CreateSession(DCSP_INIT_PARAM &iParams); - char* RunSession(cv::Mat &iImg, std::vector& oResult); + char *RunSession(cv::Mat &iImg, std::vector &oResult); - char* WarmUpSession(); + char *WarmUpSession(); - template - char* TensorProcess(clock_t& starttime_1, cv::Mat& iImg, N& blob, std::vector& inputNodeDims, std::vector& oResult); + template + char *TensorProcess(clock_t &starttime_1, cv::Mat &iImg, N &blob, std::vector &inputNodeDims, + std::vector &oResult); std::vector classes{}; private: - Ort::Env env; - Ort::Session* session; - bool cudaEnable; - Ort::RunOptions options; - std::vector inputNodeNames; - std::vector outputNodeNames; - - MODEL_TYPE modelType; - std::vector imgSize; - float rectConfidenceThreshold; - float iouThreshold; + Ort::Env env; + Ort::Session *session; + bool cudaEnable; + Ort::RunOptions options; + std::vector inputNodeNames; + std::vector outputNodeNames; + + MODEL_TYPE modelType; + std::vector imgSize; + float rectConfidenceThreshold; + float iouThreshold; }; diff --git a/examples/YOLOv8-ONNXRuntime-CPP/main.cpp b/examples/YOLOv8-ONNXRuntime-CPP/main.cpp index c2839fd..2619ba5 100644 --- a/examples/YOLOv8-ONNXRuntime-CPP/main.cpp +++ b/examples/YOLOv8-ONNXRuntime-CPP/main.cpp @@ -3,42 +3,41 @@ #include #include -void file_iterator(DCSP_CORE*& p) -{ - std::filesystem::path current_path = std::filesystem::current_path(); - std::filesystem::path imgs_path = current_path/"images"; - for (auto& i : std::filesystem::directory_iterator(imgs_path)) - { - if (i.path().extension() == ".jpg" || i.path().extension() == ".png") - { - std::string img_path = i.path().string(); - cv::Mat img = cv::imread(img_path); - std::vector res; - p->RunSession(img, res); +void file_iterator(DCSP_CORE *&p) { + std::filesystem::path current_path = std::filesystem::current_path(); + std::filesystem::path imgs_path = current_path / "images"; + for (auto &i: std::filesystem::directory_iterator(imgs_path)) { + if (i.path().extension() == ".jpg" || i.path().extension() == ".png" || i.path().extension() == ".jpeg") { + std::string img_path = i.path().string(); + cv::Mat img = cv::imread(img_path); + std::vector res; + p->RunSession(img, res); - for (auto & re : res) - { - cv::rectangle(img, re.box, cv::Scalar(0, 0 , 255), 3); - std::string label = p->classes[re.classId]; + for (auto &re: res) { + cv::RNG rng(cv::getTickCount()); + cv::Scalar color(rng.uniform(0, 256), rng.uniform(0, 256), rng.uniform(0, 256)); + + cv::rectangle(img, re.box, color, 3); + std::string label = p->classes[re.classId] + " " + std::to_string(re.confidence); cv::putText( img, label, cv::Point(re.box.x, re.box.y - 5), cv::FONT_HERSHEY_SIMPLEX, 0.75, - cv::Scalar(255, 255, 0), + color, 2 ); - } - cv::imshow("Result", img); - cv::waitKey(0); - cv::destroyAllWindows(); - } - } + } + std::cout << "Press any key to exit" << std::endl; + cv::imshow("Result of Detection", img); + cv::waitKey(0); + cv::destroyAllWindows(); + } + } } -int read_coco_yaml(DCSP_CORE*& p) -{ +int read_coco_yaml(DCSP_CORE *&p) { // Open the YAML file std::ifstream file("coco.yaml"); if (!file.is_open()) { @@ -80,17 +79,19 @@ int read_coco_yaml(DCSP_CORE*& p) } -int main() -{ - DCSP_CORE* yoloDetector = new DCSP_CORE; - std::string model_path = "yolov8n.onnx"; +int main() { + DCSP_CORE *yoloDetector = new DCSP_CORE; + std::string model_path = "yolov8n.onnx"; read_coco_yaml(yoloDetector); - // GPU FP32 inference - DCSP_INIT_PARAM params{ model_path, YOLO_ORIGIN_V8, {640, 640}, 0.1, 0.5, true }; +#ifdef USE_CUDA + // GPU FP32 inference + DCSP_INIT_PARAM params{ model_path, YOLO_ORIGIN_V8, {640, 640}, 0.1, 0.5, true }; // GPU FP16 inference - // DCSP_INIT_PARAM params{ model_path, YOLO_ORIGIN_V8_HALF, {640, 640}, 0.1, 0.5, true }; - // CPU inference - // DCSP_INIT_PARAM params{ model_path, YOLO_ORIGIN_V8, {640, 640}, 0.1, 0.5, false }; + // DCSP_INIT_PARAM params{ model_path, YOLO_ORIGIN_V8_HALF, {640, 640}, 0.1, 0.5, true }; +#else + // CPU inference + DCSP_INIT_PARAM params{model_path, YOLO_ORIGIN_V8, {640, 640}, 0.1, 0.5, false}; +#endif yoloDetector->CreateSession(params); - file_iterator(yoloDetector); + file_iterator(yoloDetector); }