xdobro23/YOLOv8-16bit
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update YOLOv8-ONNXRuntime-CPP example with GPU inference (#4328)

Browse Source 
Signed-off-by: Onuralp SEZER <thunderbirdtr@gmail.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Glenn Jocher <glenn.jocher@ultralytics.com>
This commit is contained in:
Onuralp SEZER
2023-08-13 18:34:39 +03:00
committed by GitHub
parent f6b58e9d75
commit b5d1af42d8
6 changed files with 364 additions and 362 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
examples/README.md
View File

@ -5,14 +5,14 @@ This repository features a collection of real-world applications and walkthrough
### Ultralytics YOLO Example Applications ### Ultralytics YOLO Example Applications
| Title | Format | Contributor | | Title | Format | Contributor |
| -------------------------------------------------------------------------------------------------------------- | ------------------ | --------------------------------------------------- | | -------------------------------------------------------------------------------------------------------------- | ------------------ | ----------------------------------------------------------------------------------------- |
| [YOLO ONNX Detection Inference with C++](./YOLOv8-CPP-Inference) | C++/ONNX | [Justas Bartnykas](https://github.com/JustasBart) | | [YOLO ONNX Detection Inference with C++](./YOLOv8-CPP-Inference) | C++/ONNX | [Justas Bartnykas](https://github.com/JustasBart) |
| [YOLO OpenCV ONNX Detection Python](./YOLOv8-OpenCV-ONNX-Python) | OpenCV/Python/ONNX | [Farid Inawan](https://github.com/frdteknikelektro) | | [YOLO OpenCV ONNX Detection Python](./YOLOv8-OpenCV-ONNX-Python) | OpenCV/Python/ONNX | [Farid Inawan](https://github.com/frdteknikelektro) |
| [YOLOv8 .NET ONNX ImageSharp](https://github.com/dme-compunet/YOLOv8) | C#/ONNX/ImageSharp | [Compunet](https://github.com/dme-compunet) | | [YOLOv8 .NET ONNX ImageSharp](https://github.com/dme-compunet/YOLOv8) | C#/ONNX/ImageSharp | [Compunet](https://github.com/dme-compunet) |
| [YOLO .Net ONNX Detection C#](https://www.nuget.org/packages/Yolov8.Net) | C# .Net | [Samuel Stainback](https://github.com/sstainba) | | [YOLO .Net ONNX Detection C#](https://www.nuget.org/packages/Yolov8.Net) | C# .Net | [Samuel Stainback](https://github.com/sstainba) |
| [YOLOv8 on NVIDIA Jetson(TensorRT and DeepStream)](https://wiki.seeedstudio.com/YOLOv8-DeepStream-TRT-Jetson/) | Python | [Lakshantha](https://github.com/lakshanthad) | | [YOLOv8 on NVIDIA Jetson(TensorRT and DeepStream)](https://wiki.seeedstudio.com/YOLOv8-DeepStream-TRT-Jetson/) | Python | [Lakshantha](https://github.com/lakshanthad) |
| [YOLOv8 ONNXRuntime Python](./YOLOv8-ONNXRuntime) | Python/ONNXRuntime | [Semih Demirel](https://github.com/semihhdemirel) | | [YOLOv8 ONNXRuntime Python](./YOLOv8-ONNXRuntime) | Python/ONNXRuntime | [Semih Demirel](https://github.com/semihhdemirel) |
| [YOLOv8-ONNXRuntime-CPP](./YOLOv8-ONNXRuntime-CPP) | C++/ONNXRuntime | [DennisJcy](https://github.com/DennisJcy) | | [YOLOv8-ONNXRuntime-CPP](./YOLOv8-ONNXRuntime-CPP) | C++/ONNXRuntime | [DennisJcy](https://github.com/DennisJcy), [Onuralp Sezer](https://github.com/onuralpszr) |
| [RTDETR ONNXRuntime C#](https://github.com/Kayzwer/yolo-cs/blob/master/RTDETR.cs) | C#/ONNX | [Kayzwer](https://github.com/Kayzwer) | | [RTDETR ONNXRuntime C#](https://github.com/Kayzwer/yolo-cs/blob/master/RTDETR.cs) | C#/ONNX | [Kayzwer](https://github.com/Kayzwer) |
### How to Contribute ### How to Contribute

53
examples/YOLOv8-ONNXRuntime-CPP/CMakeLists.txt
View File

@ -17,32 +17,37 @@ include_directories(${OpenCV_INCLUDE_DIRS})
# -------------- Compile CUDA for FP16 inference if needed ------------------# # -------------- Compile CUDA for FP16 inference if needed ------------------#
find_package(CUDA REQUIRED) option(USE_CUDA "Enable CUDA support" ON)
include_directories(${CUDA_INCLUDE_DIRS}) if (USE_CUDA)
find_package(CUDA REQUIRED)
include_directories(${CUDA_INCLUDE_DIRS})
add_definitions(-DUSE_CUDA)
endif ()
# ONNXRUNTIME # ONNXRUNTIME
# Set ONNXRUNTIME_VERSION # Set ONNXRUNTIME_VERSION
set(ONNXRUNTIME_VERSION 1.15.1) set(ONNXRUNTIME_VERSION 1.15.1)
if(WIN32) if (WIN32)
# CPU if (USE_CUDA)
# set(ONNXRUNTIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime-win-x64-${ONNXRUNTIME_VERSION}")
# GPU
set(ONNXRUNTIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime-win-x64-gpu-${ONNXRUNTIME_VERSION}") set(ONNXRUNTIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime-win-x64-gpu-${ONNXRUNTIME_VERSION}")
elseif(LINUX) else ()
# CPU set(ONNXRUNTIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime-win-x64-${ONNXRUNTIME_VERSION}")
# set(ONNXRUNTIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}") endif ()
# GPU elseif (LINUX)
if (USE_CUDA)
set(ONNXRUNTIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime-linux-x64-gpu-${ONNXRUNTIME_VERSION}") set(ONNXRUNTIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime-linux-x64-gpu-${ONNXRUNTIME_VERSION}")
elseif(APPLE) else ()
set(ONNXRUNTIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}")
endif ()
elseif (APPLE)
set(ONNXRUNTIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime-osx-arm64-${ONNXRUNTIME_VERSION}") set(ONNXRUNTIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime-osx-arm64-${ONNXRUNTIME_VERSION}")
# Apple X64 binary # Apple X64 binary
# set(ONNXRUNTIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime-osx-x64-${ONNXRUNTIME_VERSION}") # set(ONNXRUNTIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime-osx-x64-${ONNXRUNTIME_VERSION}")
# Apple Universal binary # Apple Universal binary
# set(ONNXRUNTIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime-osx-universal2-${ONNXRUNTIME_VERSION}") # set(ONNXRUNTIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime-osx-universal2-${ONNXRUNTIME_VERSION}")
endif() endif ()
include_directories(${PROJECT_NAME} ${ONNXRUNTIME_ROOT}/include) include_directories(${PROJECT_NAME} ${ONNXRUNTIME_ROOT}/include)
@ -54,21 +59,27 @@ set(PROJECT_SOURCES
add_executable(${PROJECT_NAME} ${PROJECT_SOURCES}) add_executable(${PROJECT_NAME} ${PROJECT_SOURCES})
if(WIN32) if (WIN32)
target_link_libraries(${PROJECT_NAME} ${OpenCV_LIBS} ${ONNXRUNTIME_ROOT}/lib/onnxruntime.lib ${CUDA_LIBRARIES}) target_link_libraries(${PROJECT_NAME} ${OpenCV_LIBS} ${ONNXRUNTIME_ROOT}/lib/onnxruntime.lib)
elseif(LINUX) if (USE_CUDA)
target_link_libraries(${PROJECT_NAME} ${OpenCV_LIBS} ${ONNXRUNTIME_ROOT}/lib/libonnxruntime.so ${CUDA_LIBRARIES}) target_link_libraries(${PROJECT_NAME} ${CUDA_LIBRARIES})
elseif(APPLE) endif ()
elseif (LINUX)
target_link_libraries(${PROJECT_NAME} ${OpenCV_LIBS} ${ONNXRUNTIME_ROOT}/lib/libonnxruntime.so)
if (USE_CUDA)
target_link_libraries(${PROJECT_NAME} ${CUDA_LIBRARIES})
endif ()
elseif (APPLE)
target_link_libraries(${PROJECT_NAME} ${OpenCV_LIBS} ${ONNXRUNTIME_ROOT}/lib/libonnxruntime.dylib) target_link_libraries(${PROJECT_NAME} ${OpenCV_LIBS} ${ONNXRUNTIME_ROOT}/lib/libonnxruntime.dylib)
endif() endif ()
# For windows system, copy onnxruntime.dll to the same folder of the executable file # For windows system, copy onnxruntime.dll to the same folder of the executable file
if(WIN32) if (WIN32)
add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_if_different COMMAND ${CMAKE_COMMAND} -E copy_if_different
"${ONNXRUNTIME_ROOT}/lib/onnxruntime.dll" "${ONNXRUNTIME_ROOT}/lib/onnxruntime.dll"
$<TARGET_FILE_DIR:${PROJECT_NAME}>) $<TARGET_FILE_DIR:${PROJECT_NAME}>)
endif() endif ()
# Download https://raw.githubusercontent.com/ultralytics/ultralytics/main/ultralytics/cfg/datasets/coco.yaml # Download https://raw.githubusercontent.com/ultralytics/ultralytics/main/ultralytics/cfg/datasets/coco.yaml
# and put it in the same folder of the executable file # and put it in the same folder of the executable file

9
examples/YOLOv8-ONNXRuntime-CPP/README.md
View File

@ -28,16 +28,23 @@ Alternatively, you can use the following command for exporting the model in the
yolo export model=yolov8n.pt opset=12 simplify=True dynamic=False format=onnx imgsz=640,640 yolo export model=yolov8n.pt opset=12 simplify=True dynamic=False format=onnx imgsz=640,640
``` ```
## Download COCO.yaml file
In order to run example, you also need to download coco.yaml. You can download the file manually from [here](https://raw.githubusercontent.com/ultralytics/ultralytics/main/ultralytics/cfg/datasets/coco.yaml)
## Dependencies ## Dependencies
| Dependency | Version | | Dependency | Version |
| -------------------------------- | -------- | | -------------------------------- | ------------- |
| Onnxruntime(linux,windows,macos) | >=1.14.1 | | Onnxruntime(linux,windows,macos) | >=1.14.1 |
| OpenCV | >=4.0.0 | | OpenCV | >=4.0.0 |
| C++ | >=17 | | C++ | >=17 |
| Cmake | >=3.5 | | Cmake | >=3.5 |
| Cuda (Optional) | >=11.4,\<12.0 |
| cuDNN (Cuda required) | =8 |
Note: The dependency on C++17 is due to the usage of the C++17 filesystem feature. Note: The dependency on C++17 is due to the usage of the C++17 filesystem feature.
Note (2): Due to ONNX Runtime, we need to use CUDA 11 and cuDNN 8. Keep in mind that this requirement might change in the future.
## Usage ## Usage

161
examples/YOLOv8-ONNXRuntime-CPP/inference.cpp
View File

@ -3,39 +3,35 @@
#define benchmark #define benchmark
DCSP_CORE::DCSP_CORE() DCSP_CORE::DCSP_CORE() {
{
} }
DCSP_CORE::~DCSP_CORE() DCSP_CORE::~DCSP_CORE() {
{
delete session; delete session;
} }
#ifdef USE_CUDA
namespace Ort namespace Ort
{ {
template<> template<>
struct TypeToTensorType<half> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16; }; struct TypeToTensorType<half> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16; };
} }
#endif
template<typename T> template<typename T>
char* BlobFromImage(cv::Mat& iImg, T& iBlob) char *BlobFromImage(cv::Mat &iImg, T &iBlob) {
{
int channels = iImg.channels(); int channels = iImg.channels();
int imgHeight = iImg.rows; int imgHeight = iImg.rows;
int imgWidth = iImg.cols; int imgWidth = iImg.cols;
for (int c = 0; c < channels; c++) for (int c = 0; c < channels; c++) {
{ for (int h = 0; h < imgHeight; h++) {
for (int h = 0; h < imgHeight; h++) for (int w = 0; w < imgWidth; w++) {
{ iBlob[c * imgWidth * imgHeight + h * imgWidth + w] = typename std::remove_pointer<T>::type(
for (int w = 0; w < imgWidth; w++) (iImg.at<cv::Vec3b>(h, w)[c]) / 255.0f);
{
iBlob[c * imgWidth * imgHeight + h * imgWidth + w] = typename std::remove_pointer<T>::type((iImg.at<cv::Vec3b>(h, w)[c]) / 255.0f);
} }
} }
} }
@ -43,12 +39,10 @@ char* BlobFromImage(cv::Mat& iImg, T& iBlob)
} }
char* PostProcess(cv::Mat& iImg, std::vector<int> iImgSize, cv::Mat& oImg) char *PostProcess(cv::Mat &iImg, std::vector<int> iImgSize, cv::Mat &oImg) {
{
cv::Mat img = iImg.clone(); cv::Mat img = iImg.clone();
cv::resize(iImg, oImg, cv::Size(iImgSize.at(0), iImgSize.at(1))); cv::resize(iImg, oImg, cv::Size(iImgSize.at(0), iImgSize.at(1)));
if (img.channels() == 1) if (img.channels() == 1) {
{
cv::cvtColor(oImg, oImg, cv::COLOR_GRAY2BGR); cv::cvtColor(oImg, oImg, cv::COLOR_GRAY2BGR);
} }
cv::cvtColor(oImg, oImg, cv::COLOR_BGR2RGB); cv::cvtColor(oImg, oImg, cv::COLOR_BGR2RGB);
@ -56,27 +50,23 @@ char* PostProcess(cv::Mat& iImg, std::vector<int> iImgSize, cv::Mat& oImg)
} }
char* DCSP_CORE::CreateSession(DCSP_INIT_PARAM &iParams) char *DCSP_CORE::CreateSession(DCSP_INIT_PARAM &iParams) {
{ char *Ret = RET_OK;
char* Ret = RET_OK;
std::regex pattern("[\u4e00-\u9fa5]"); std::regex pattern("[\u4e00-\u9fa5]");
bool result = std::regex_search(iParams.ModelPath, pattern); bool result = std::regex_search(iParams.ModelPath, pattern);
if (result) if (result) {
{
Ret = "[DCSP_ONNX]:Model path error.Change your model path without chinese characters."; Ret = "[DCSP_ONNX]:Model path error.Change your model path without chinese characters.";
std::cout << Ret << std::endl; std::cout << Ret << std::endl;
return Ret; return Ret;
} }
try try {
{
rectConfidenceThreshold = iParams.RectConfidenceThreshold; rectConfidenceThreshold = iParams.RectConfidenceThreshold;
iouThreshold = iParams.iouThreshold; iouThreshold = iParams.iouThreshold;
imgSize = iParams.imgSize; imgSize = iParams.imgSize;
modelType = iParams.ModelType; modelType = iParams.ModelType;
env = Ort::Env(ORT_LOGGING_LEVEL_WARNING, "Yolo"); env = Ort::Env(ORT_LOGGING_LEVEL_WARNING, "Yolo");
Ort::SessionOptions sessionOption; Ort::SessionOptions sessionOption;
if (iParams.CudaEnable) if (iParams.CudaEnable) {
{
cudaEnable = iParams.CudaEnable; cudaEnable = iParams.CudaEnable;
OrtCUDAProviderOptions cudaOption; OrtCUDAProviderOptions cudaOption;
cudaOption.device_id = 0; cudaOption.device_id = 0;
@ -93,37 +83,34 @@ char* DCSP_CORE::CreateSession(DCSP_INIT_PARAM &iParams)
wide_cstr[ModelPathSize] = L'\0'; wide_cstr[ModelPathSize] = L'\0';
const wchar_t* modelPath = wide_cstr; const wchar_t* modelPath = wide_cstr;
#else #else
const char* modelPath = iParams.ModelPath.c_str(); const char *modelPath = iParams.ModelPath.c_str();
#endif // _WIN32 #endif // _WIN32
session = new Ort::Session(env, modelPath, sessionOption); session = new Ort::Session(env, modelPath, sessionOption);
Ort::AllocatorWithDefaultOptions allocator; Ort::AllocatorWithDefaultOptions allocator;
size_t inputNodesNum = session->GetInputCount(); size_t inputNodesNum = session->GetInputCount();
for (size_t i = 0; i < inputNodesNum; i++) for (size_t i = 0; i < inputNodesNum; i++) {
{
Ort::AllocatedStringPtr input_node_name = session->GetInputNameAllocated(i, allocator); Ort::AllocatedStringPtr input_node_name = session->GetInputNameAllocated(i, allocator);
char* temp_buf = new char[50]; char *temp_buf = new char[50];
strcpy(temp_buf, input_node_name.get()); strcpy(temp_buf, input_node_name.get());
inputNodeNames.push_back(temp_buf); inputNodeNames.push_back(temp_buf);
} }
size_t OutputNodesNum = session->GetOutputCount(); size_t OutputNodesNum = session->GetOutputCount();
for (size_t i = 0; i < OutputNodesNum; i++) for (size_t i = 0; i < OutputNodesNum; i++) {
{
Ort::AllocatedStringPtr output_node_name = session->GetOutputNameAllocated(i, allocator); Ort::AllocatedStringPtr output_node_name = session->GetOutputNameAllocated(i, allocator);
char* temp_buf = new char[10]; char *temp_buf = new char[10];
strcpy(temp_buf, output_node_name.get()); strcpy(temp_buf, output_node_name.get());
outputNodeNames.push_back(temp_buf); outputNodeNames.push_back(temp_buf);
} }
options = Ort::RunOptions{ nullptr }; options = Ort::RunOptions{nullptr};
WarmUpSession(); WarmUpSession();
return RET_OK; return RET_OK;
} }
catch (const std::exception& e) catch (const std::exception &e) {
{ const char *str1 = "[DCSP_ONNX]:";
const char* str1 = "[DCSP_ONNX]:"; const char *str2 = e.what();
const char* str2 = e.what();
std::string result = std::string(str1) + std::string(str2); std::string result = std::string(str1) + std::string(str2);
char* merged = new char[result.length() + 1]; char *merged = new char[result.length() + 1];
std::strcpy(merged, result.c_str()); std::strcpy(merged, result.c_str());
std::cout << merged << std::endl; std::cout << merged << std::endl;
delete[] merged; delete[] merged;
@ -133,28 +120,26 @@ char* DCSP_CORE::CreateSession(DCSP_INIT_PARAM &iParams)
} }
char* DCSP_CORE::RunSession(cv::Mat &iImg, std::vector<DCSP_RESULT>& oResult) char *DCSP_CORE::RunSession(cv::Mat &iImg, std::vector<DCSP_RESULT> &oResult) {
{
#ifdef benchmark #ifdef benchmark
clock_t starttime_1 = clock(); clock_t starttime_1 = clock();
#endif // benchmark #endif // benchmark
char* Ret = RET_OK; char *Ret = RET_OK;
cv::Mat processedImg; cv::Mat processedImg;
PostProcess(iImg, imgSize, processedImg); PostProcess(iImg, imgSize, processedImg);
if (modelType < 4) if (modelType < 4) {
{ float *blob = new float[processedImg.total() * 3];
float* blob = new float[processedImg.total() * 3];
BlobFromImage(processedImg, blob); BlobFromImage(processedImg, blob);
std::vector<int64_t> inputNodeDims = { 1,3,imgSize.at(0),imgSize.at(1) }; std::vector<int64_t> inputNodeDims = {1, 3, imgSize.at(0), imgSize.at(1)};
TensorProcess(starttime_1, iImg, blob, inputNodeDims, oResult); TensorProcess(starttime_1, iImg, blob, inputNodeDims, oResult);
} } else {
else #ifdef USE_CUDA
{
half* blob = new half[processedImg.total() * 3]; half* blob = new half[processedImg.total() * 3];
BlobFromImage(processedImg, blob); BlobFromImage(processedImg, blob);
std::vector<int64_t> inputNodeDims = { 1,3,imgSize.at(0),imgSize.at(1) }; std::vector<int64_t> inputNodeDims = { 1,3,imgSize.at(0),imgSize.at(1) };
TensorProcess(starttime_1, iImg, blob, inputNodeDims, oResult); TensorProcess(starttime_1, iImg, blob, inputNodeDims, oResult);
#endif
} }
return Ret; return Ret;
@ -162,24 +147,26 @@ char* DCSP_CORE::RunSession(cv::Mat &iImg, std::vector<DCSP_RESULT>& oResult)
template<typename N> template<typename N>
char* DCSP_CORE::TensorProcess(clock_t& starttime_1, cv::Mat& iImg, N& blob, std::vector<int64_t>& inputNodeDims, std::vector<DCSP_RESULT>& oResult) char *DCSP_CORE::TensorProcess(clock_t &starttime_1, cv::Mat &iImg, N &blob, std::vector<int64_t> &inputNodeDims,
{ std::vector<DCSP_RESULT> &oResult) {
Ort::Value inputTensor = Ort::Value::CreateTensor<typename std::remove_pointer<N>::type>(Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU), blob, 3 * imgSize.at(0) * imgSize.at(1), inputNodeDims.data(), inputNodeDims.size()); Ort::Value inputTensor = Ort::Value::CreateTensor<typename std::remove_pointer<N>::type>(
Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU), blob, 3 * imgSize.at(0) * imgSize.at(1),
inputNodeDims.data(), inputNodeDims.size());
#ifdef benchmark #ifdef benchmark
clock_t starttime_2 = clock(); clock_t starttime_2 = clock();
#endif // benchmark #endif // benchmark
auto outputTensor = session->Run(options, inputNodeNames.data(), &inputTensor, 1, outputNodeNames.data(), outputNodeNames.size()); auto outputTensor = session->Run(options, inputNodeNames.data(), &inputTensor, 1, outputNodeNames.data(),
outputNodeNames.size());
#ifdef benchmark #ifdef benchmark
clock_t starttime_3 = clock(); clock_t starttime_3 = clock();
#endif // benchmark #endif // benchmark
Ort::TypeInfo typeInfo = outputTensor.front().GetTypeInfo(); Ort::TypeInfo typeInfo = outputTensor.front().GetTypeInfo();
auto tensor_info = typeInfo.GetTensorTypeAndShapeInfo(); auto tensor_info = typeInfo.GetTensorTypeAndShapeInfo();
std::vector<int64_t>outputNodeDims = tensor_info.GetShape(); std::vector<int64_t> outputNodeDims = tensor_info.GetShape();
auto output = outputTensor.front().GetTensorMutableData<typename std::remove_pointer<N>::type>(); auto output = outputTensor.front().GetTensorMutableData<typename std::remove_pointer<N>::type>();
delete blob; delete blob;
switch (modelType) switch (modelType) {
{
case 1://V8_ORIGIN_FP32 case 1://V8_ORIGIN_FP32
case 4://V8_ORIGIN_FP16 case 4://V8_ORIGIN_FP16
{ {
@ -191,19 +178,17 @@ char* DCSP_CORE::TensorProcess(clock_t& starttime_1, cv::Mat& iImg, N& blob, std
cv::Mat rowData(signalResultNum, strideNum, CV_32F, output); cv::Mat rowData(signalResultNum, strideNum, CV_32F, output);
rowData = rowData.t(); rowData = rowData.t();
float* data = (float*)rowData.data; float *data = (float *) rowData.data;
float x_factor = iImg.cols / 640.; float x_factor = iImg.cols / 640.;
float y_factor = iImg.rows / 640.; float y_factor = iImg.rows / 640.;
for (int i = 0; i < strideNum; ++i) for (int i = 0; i < strideNum; ++i) {
{ float *classesScores = data + 4;
float* classesScores = data + 4;
cv::Mat scores(1, this->classes.size(), CV_32FC1, classesScores); cv::Mat scores(1, this->classes.size(), CV_32FC1, classesScores);
cv::Point class_id; cv::Point class_id;
double maxClassScore; double maxClassScore;
cv::minMaxLoc(scores, 0, &maxClassScore, 0, &class_id); cv::minMaxLoc(scores, 0, &maxClassScore, 0, &class_id);
if (maxClassScore > rectConfidenceThreshold) if (maxClassScore > rectConfidenceThreshold) {
{
confidences.push_back(maxClassScore); confidences.push_back(maxClassScore);
class_ids.push_back(class_id.x); class_ids.push_back(class_id.x);
@ -226,8 +211,7 @@ char* DCSP_CORE::TensorProcess(clock_t& starttime_1, cv::Mat& iImg, N& blob, std
std::vector<int> nmsResult; std::vector<int> nmsResult;
cv::dnn::NMSBoxes(boxes, confidences, rectConfidenceThreshold, iouThreshold, nmsResult); cv::dnn::NMSBoxes(boxes, confidences, rectConfidenceThreshold, iouThreshold, nmsResult);
for (int i = 0; i < nmsResult.size(); ++i) for (int i = 0; i < nmsResult.size(); ++i) {
{
int idx = nmsResult[i]; int idx = nmsResult[i];
DCSP_RESULT result; DCSP_RESULT result;
result.classId = class_ids[idx]; result.classId = class_ids[idx];
@ -239,16 +223,15 @@ char* DCSP_CORE::TensorProcess(clock_t& starttime_1, cv::Mat& iImg, N& blob, std
#ifdef benchmark #ifdef benchmark
clock_t starttime_4 = clock(); clock_t starttime_4 = clock();
double pre_process_time = (double)(starttime_2 - starttime_1) / CLOCKS_PER_SEC * 1000; double pre_process_time = (double) (starttime_2 - starttime_1) / CLOCKS_PER_SEC * 1000;
double process_time = (double)(starttime_3 - starttime_2) / CLOCKS_PER_SEC * 1000; double process_time = (double) (starttime_3 - starttime_2) / CLOCKS_PER_SEC * 1000;
double post_process_time = (double)(starttime_4 - starttime_3) / CLOCKS_PER_SEC * 1000; double post_process_time = (double) (starttime_4 - starttime_3) / CLOCKS_PER_SEC * 1000;
if (cudaEnable) if (cudaEnable) {
{ std::cout << "[DCSP_ONNX(CUDA)]: " << pre_process_time << "ms pre-process, " << process_time
std::cout << "[DCSP_ONNX(CUDA)]: " << pre_process_time << "ms pre-process, " << process_time << "ms inference, " << post_process_time << "ms post-process." << std::endl; << "ms inference, " << post_process_time << "ms post-process." << std::endl;
} } else {
else std::cout << "[DCSP_ONNX(CPU)]: " << pre_process_time << "ms pre-process, " << process_time
{ << "ms inference, " << post_process_time << "ms post-process." << std::endl;
std::cout << "[DCSP_ONNX(CPU)]: " << pre_process_time << "ms pre-process, " << process_time << "ms inference, " << post_process_time << "ms post-process." << std::endl;
} }
#endif // benchmark #endif // benchmark
@ -259,29 +242,28 @@ char* DCSP_CORE::TensorProcess(clock_t& starttime_1, cv::Mat& iImg, N& blob, std
} }
char* DCSP_CORE::WarmUpSession() char *DCSP_CORE::WarmUpSession() {
{
clock_t starttime_1 = clock(); clock_t starttime_1 = clock();
cv::Mat iImg = cv::Mat(cv::Size(imgSize.at(0), imgSize.at(1)), CV_8UC3); cv::Mat iImg = cv::Mat(cv::Size(imgSize.at(0), imgSize.at(1)), CV_8UC3);
cv::Mat processedImg; cv::Mat processedImg;
PostProcess(iImg, imgSize, processedImg); PostProcess(iImg, imgSize, processedImg);
if (modelType < 4) if (modelType < 4) {
{ float *blob = new float[iImg.total() * 3];
float* blob = new float[iImg.total() * 3];
BlobFromImage(processedImg, blob); BlobFromImage(processedImg, blob);
std::vector<int64_t> YOLO_input_node_dims = { 1,3,imgSize.at(0),imgSize.at(1) }; std::vector<int64_t> YOLO_input_node_dims = {1, 3, imgSize.at(0), imgSize.at(1)};
Ort::Value input_tensor = Ort::Value::CreateTensor<float>(Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU), blob, 3 * imgSize.at(0) * imgSize.at(1), YOLO_input_node_dims.data(), YOLO_input_node_dims.size()); Ort::Value input_tensor = Ort::Value::CreateTensor<float>(
auto output_tensors = session->Run(options, inputNodeNames.data(), &input_tensor, 1, outputNodeNames.data(), outputNodeNames.size()); Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU), blob, 3 * imgSize.at(0) * imgSize.at(1),
YOLO_input_node_dims.data(), YOLO_input_node_dims.size());
auto output_tensors = session->Run(options, inputNodeNames.data(), &input_tensor, 1, outputNodeNames.data(),
outputNodeNames.size());
delete[] blob; delete[] blob;
clock_t starttime_4 = clock(); clock_t starttime_4 = clock();
double post_process_time = (double)(starttime_4 - starttime_1) / CLOCKS_PER_SEC * 1000; double post_process_time = (double) (starttime_4 - starttime_1) / CLOCKS_PER_SEC * 1000;
if (cudaEnable) if (cudaEnable) {
{
std::cout << "[DCSP_ONNX(CUDA)]: " << "Cuda warm-up cost " << post_process_time << " ms. " << std::endl; std::cout << "[DCSP_ONNX(CUDA)]: " << "Cuda warm-up cost " << post_process_time << " ms. " << std::endl;
} }
} } else {
else #ifdef USE_CUDA
{
half* blob = new half[iImg.total() * 3]; half* blob = new half[iImg.total() * 3];
BlobFromImage(processedImg, blob); BlobFromImage(processedImg, blob);
std::vector<int64_t> YOLO_input_node_dims = { 1,3,imgSize.at(0),imgSize.at(1) }; std::vector<int64_t> YOLO_input_node_dims = { 1,3,imgSize.at(0),imgSize.at(1) };
@ -294,6 +276,7 @@ char* DCSP_CORE::WarmUpSession()
{ {
std::cout << "[DCSP_ONNX(CUDA)]: " << "Cuda warm-up cost " << post_process_time << " ms. " << std::endl; std::cout << "[DCSP_ONNX(CUDA)]: " << "Cuda warm-up cost " << post_process_time << " ms. " << std::endl;
} }
#endif
} }
return RET_OK; return RET_OK;
} }

38
examples/YOLOv8-ONNXRuntime-CPP/inference.h
View File

@ -13,11 +13,13 @@
#include <cstdio> #include <cstdio>
#include <opencv2/opencv.hpp> #include <opencv2/opencv.hpp>
#include "onnxruntime_cxx_api.h" #include "onnxruntime_cxx_api.h"
#ifdef USE_CUDA
#include <cuda_fp16.h> #include <cuda_fp16.h>
#endif
enum MODEL_TYPE enum MODEL_TYPE {
{
//FLOAT32 MODEL //FLOAT32 MODEL
YOLO_ORIGIN_V5 = 0, YOLO_ORIGIN_V5 = 0,
YOLO_ORIGIN_V8 = 1,//only support v8 detector currently YOLO_ORIGIN_V8 = 1,//only support v8 detector currently
@ -29,53 +31,51 @@ enum MODEL_TYPE
}; };
typedef struct _DCSP_INIT_PARAM {
typedef struct _DCSP_INIT_PARAM
{
std::string ModelPath; std::string ModelPath;
MODEL_TYPE ModelType = YOLO_ORIGIN_V8; MODEL_TYPE ModelType = YOLO_ORIGIN_V8;
std::vector<int> imgSize={640, 640}; std::vector<int> imgSize = {640, 640};
float RectConfidenceThreshold = 0.6; float RectConfidenceThreshold = 0.6;
float iouThreshold = 0.5; float iouThreshold = 0.5;
bool CudaEnable = false; bool CudaEnable = false;
int LogSeverityLevel = 3; int LogSeverityLevel = 3;
int IntraOpNumThreads = 1; int IntraOpNumThreads = 1;
}DCSP_INIT_PARAM; } DCSP_INIT_PARAM;
typedef struct _DCSP_RESULT typedef struct _DCSP_RESULT {
{
int classId; int classId;
float confidence; float confidence;
cv::Rect box; cv::Rect box;
}DCSP_RESULT; } DCSP_RESULT;
class DCSP_CORE class DCSP_CORE {
{
public: public:
DCSP_CORE(); DCSP_CORE();
~DCSP_CORE(); ~DCSP_CORE();
public: public:
char* CreateSession(DCSP_INIT_PARAM &iParams); char *CreateSession(DCSP_INIT_PARAM &iParams);
char* RunSession(cv::Mat &iImg, std::vector<DCSP_RESULT>& oResult); char *RunSession(cv::Mat &iImg, std::vector<DCSP_RESULT> &oResult);
char* WarmUpSession(); char *WarmUpSession();
template<typename N> template<typename N>
char* TensorProcess(clock_t& starttime_1, cv::Mat& iImg, N& blob, std::vector<int64_t>& inputNodeDims, std::vector<DCSP_RESULT>& oResult); char *TensorProcess(clock_t &starttime_1, cv::Mat &iImg, N &blob, std::vector<int64_t> &inputNodeDims,
std::vector<DCSP_RESULT> &oResult);
std::vector<std::string> classes{}; std::vector<std::string> classes{};
private: private:
Ort::Env env; Ort::Env env;
Ort::Session* session; Ort::Session *session;
bool cudaEnable; bool cudaEnable;
Ort::RunOptions options; Ort::RunOptions options;
std::vector<const char*> inputNodeNames; std::vector<const char *> inputNodeNames;
std::vector<const char*> outputNodeNames; std::vector<const char *> outputNodeNames;
MODEL_TYPE modelType; MODEL_TYPE modelType;
std::vector<int> imgSize; std::vector<int> imgSize;

39
examples/YOLOv8-ONNXRuntime-CPP/main.cpp
View File

@ -3,42 +3,41 @@
#include <filesystem> #include <filesystem>
#include <fstream> #include <fstream>
void file_iterator(DCSP_CORE*& p) void file_iterator(DCSP_CORE *&p) {
{
std::filesystem::path current_path = std::filesystem::current_path(); std::filesystem::path current_path = std::filesystem::current_path();
std::filesystem::path imgs_path = current_path/"images"; std::filesystem::path imgs_path = current_path / "images";
for (auto& i : std::filesystem::directory_iterator(imgs_path)) for (auto &i: std::filesystem::directory_iterator(imgs_path)) {
{ if (i.path().extension() == ".jpg" || i.path().extension() == ".png" || i.path().extension() == ".jpeg") {
if (i.path().extension() == ".jpg" || i.path().extension() == ".png")
{
std::string img_path = i.path().string(); std::string img_path = i.path().string();
cv::Mat img = cv::imread(img_path); cv::Mat img = cv::imread(img_path);
std::vector<DCSP_RESULT> res; std::vector<DCSP_RESULT> res;
p->RunSession(img, res); p->RunSession(img, res);
for (auto & re : res) for (auto &re: res) {
{ cv::RNG rng(cv::getTickCount());
cv::rectangle(img, re.box, cv::Scalar(0, 0 , 255), 3); cv::Scalar color(rng.uniform(0, 256), rng.uniform(0, 256), rng.uniform(0, 256));
std::string label = p->classes[re.classId];
cv::rectangle(img, re.box, color, 3);
std::string label = p->classes[re.classId] + " " + std::to_string(re.confidence);
cv::putText( cv::putText(
img, img,
label, label,
cv::Point(re.box.x, re.box.y - 5), cv::Point(re.box.x, re.box.y - 5),
cv::FONT_HERSHEY_SIMPLEX, cv::FONT_HERSHEY_SIMPLEX,
0.75, 0.75,
cv::Scalar(255, 255, 0), color,
2 2
); );
} }
cv::imshow("Result", img); std::cout << "Press any key to exit" << std::endl;
cv::imshow("Result of Detection", img);
cv::waitKey(0); cv::waitKey(0);
cv::destroyAllWindows(); cv::destroyAllWindows();
} }
} }
} }
int read_coco_yaml(DCSP_CORE*& p) int read_coco_yaml(DCSP_CORE *&p) {
{
// Open the YAML file // Open the YAML file
std::ifstream file("coco.yaml"); std::ifstream file("coco.yaml");
if (!file.is_open()) { if (!file.is_open()) {
@ -80,17 +79,19 @@ int read_coco_yaml(DCSP_CORE*& p)
} }
int main() int main() {
{ DCSP_CORE *yoloDetector = new DCSP_CORE;
DCSP_CORE* yoloDetector = new DCSP_CORE;
std::string model_path = "yolov8n.onnx"; std::string model_path = "yolov8n.onnx";
read_coco_yaml(yoloDetector); read_coco_yaml(yoloDetector);
#ifdef USE_CUDA
// GPU FP32 inference // GPU FP32 inference
DCSP_INIT_PARAM params{ model_path, YOLO_ORIGIN_V8, {640, 640}, 0.1, 0.5, true }; DCSP_INIT_PARAM params{ model_path, YOLO_ORIGIN_V8, {640, 640}, 0.1, 0.5, true };
// GPU FP16 inference // GPU FP16 inference
// DCSP_INIT_PARAM params{ model_path, YOLO_ORIGIN_V8_HALF, {640, 640}, 0.1, 0.5, true }; // DCSP_INIT_PARAM params{ model_path, YOLO_ORIGIN_V8_HALF, {640, 640}, 0.1, 0.5, true };
#else
// CPU inference // CPU inference
// DCSP_INIT_PARAM params{ model_path, YOLO_ORIGIN_V8, {640, 640}, 0.1, 0.5, false }; DCSP_INIT_PARAM params{model_path, YOLO_ORIGIN_V8, {640, 640}, 0.1, 0.5, false};
#endif
yoloDetector->CreateSession(params); yoloDetector->CreateSession(params);
file_iterator(yoloDetector); file_iterator(yoloDetector);
} }