diff --git a/docs/modes/benchmark.md b/docs/modes/benchmark.md index c1c159e..0c5b35f 100644 --- a/docs/modes/benchmark.md +++ b/docs/modes/benchmark.md @@ -70,5 +70,6 @@ Benchmarks will attempt to run automatically on all possible export formats belo | [TF Edge TPU](https://coral.ai/docs/edgetpu/models-intro/) | `edgetpu` | `yolov8n_edgetpu.tflite` | ✅ | | [TF.js](https://www.tensorflow.org/js) | `tfjs` | `yolov8n_web_model/` | ✅ | | [PaddlePaddle](https://github.com/PaddlePaddle) | `paddle` | `yolov8n_paddle_model/` | ✅ | +| [ncnn](https://github.com/Tencent/ncnn) | `ncnn` | `yolov8n_ncnn_model/` | ✅ | See full `export` details in the [Export](https://docs.ultralytics.com/modes/export/) page. \ No newline at end of file diff --git a/docs/modes/export.md b/docs/modes/export.md index 54518b2..bff65d3 100644 --- a/docs/modes/export.md +++ b/docs/modes/export.md @@ -1,7 +1,7 @@ --- comments: true description: 'Export mode: Create a deployment-ready YOLOv8 model by converting it to various formats. Export to ONNX or OpenVINO for up to 3x CPU speedup.' -keywords: ultralytics docs, YOLOv8, export YOLOv8, YOLOv8 model deployment, exporting YOLOv8, ONNX, OpenVINO, TensorRT, CoreML, TF SavedModel, PaddlePaddle, TorchScript, ONNX format, OpenVINO format, TensorRT format, CoreML format, TF SavedModel format, PaddlePaddle format, Tencent NCNN, NCNN +keywords: ultralytics docs, YOLOv8, export YOLOv8, YOLOv8 model deployment, exporting YOLOv8, ONNX, OpenVINO, TensorRT, CoreML, TF SavedModel, PaddlePaddle, TorchScript, ONNX format, OpenVINO format, TensorRT format, CoreML format, TF SavedModel format, PaddlePaddle format, Tencent ncnn format --- @@ -85,4 +85,4 @@ i.e. `format='onnx'` or `format='engine'`. | [TF Edge TPU](https://coral.ai/docs/edgetpu/models-intro/) | `edgetpu` | `yolov8n_edgetpu.tflite` | ✅ | `imgsz` | | [TF.js](https://www.tensorflow.org/js) | `tfjs` | `yolov8n_web_model/` | ✅ | `imgsz` | | [PaddlePaddle](https://github.com/PaddlePaddle) | `paddle` | `yolov8n_paddle_model/` | ✅ | `imgsz` | -| [NCNN](https://github.com/Tencent/ncnn) | `ncnn` | `yolov8n_ncnn_model/` | ✅ | `imgsz`, `half` | \ No newline at end of file +| [ncnn](https://github.com/Tencent/ncnn) | `ncnn` | `yolov8n_ncnn_model/` | ✅ | `imgsz`, `half` | \ No newline at end of file diff --git a/docs/tasks/classify.md b/docs/tasks/classify.md index cfcf387..8d71093 100644 --- a/docs/tasks/classify.md +++ b/docs/tasks/classify.md @@ -176,6 +176,6 @@ i.e. `yolo predict model=yolov8n-cls.onnx`. Usage examples are shown for your mo | [TF Edge TPU](https://coral.ai/docs/edgetpu/models-intro/) | `edgetpu` | `yolov8n-cls_edgetpu.tflite` | ✅ | `imgsz` | | [TF.js](https://www.tensorflow.org/js) | `tfjs` | `yolov8n-cls_web_model/` | ✅ | `imgsz` | | [PaddlePaddle](https://github.com/PaddlePaddle) | `paddle` | `yolov8n-cls_paddle_model/` | ✅ | `imgsz` | -| [NCNN](https://github.com/Tencent/ncnn) | `ncnn` | `yolov8n-cls_ncnn_model/` | ✅ | `imgsz`, `half` | +| [ncnn](https://github.com/Tencent/ncnn) | `ncnn` | `yolov8n-cls_ncnn_model/` | ✅ | `imgsz`, `half` | See full `export` details in the [Export](https://docs.ultralytics.com/modes/export/) page. \ No newline at end of file diff --git a/docs/tasks/detect.md b/docs/tasks/detect.md index 18d2a6f..6573b75 100644 --- a/docs/tasks/detect.md +++ b/docs/tasks/detect.md @@ -167,6 +167,6 @@ Available YOLOv8 export formats are in the table below. You can predict or valid | [TF Edge TPU](https://coral.ai/docs/edgetpu/models-intro/) | `edgetpu` | `yolov8n_edgetpu.tflite` | ✅ | `imgsz` | | [TF.js](https://www.tensorflow.org/js) | `tfjs` | `yolov8n_web_model/` | ✅ | `imgsz` | | [PaddlePaddle](https://github.com/PaddlePaddle) | `paddle` | `yolov8n_paddle_model/` | ✅ | `imgsz` | -| [NCNN](https://github.com/Tencent/ncnn) | `ncnn` | `yolov8n_ncnn_model/` | ✅ | `imgsz`, `half` | +| [ncnn](https://github.com/Tencent/ncnn) | `ncnn` | `yolov8n_ncnn_model/` | ✅ | `imgsz`, `half` | See full `export` details in the [Export](https://docs.ultralytics.com/modes/export/) page. \ No newline at end of file diff --git a/docs/tasks/pose.md b/docs/tasks/pose.md index 6ed68ae..ef8495d 100644 --- a/docs/tasks/pose.md +++ b/docs/tasks/pose.md @@ -181,6 +181,6 @@ i.e. `yolo predict model=yolov8n-pose.onnx`. Usage examples are shown for your m | [TF Edge TPU](https://coral.ai/docs/edgetpu/models-intro/) | `edgetpu` | `yolov8n-pose_edgetpu.tflite` | ✅ | `imgsz` | | [TF.js](https://www.tensorflow.org/js) | `tfjs` | `yolov8n-pose_web_model/` | ✅ | `imgsz` | | [PaddlePaddle](https://github.com/PaddlePaddle) | `paddle` | `yolov8n-pose_paddle_model/` | ✅ | `imgsz` | -| [NCNN](https://github.com/Tencent/ncnn) | `ncnn` | `yolov8n-pose_ncnn_model/` | ✅ | `imgsz`, `half` | +| [ncnn](https://github.com/Tencent/ncnn) | `ncnn` | `yolov8n-pose_ncnn_model/` | ✅ | `imgsz`, `half` | See full `export` details in the [Export](https://docs.ultralytics.com/modes/export/) page. \ No newline at end of file diff --git a/docs/tasks/segment.md b/docs/tasks/segment.md index 586bcfc..659ac82 100644 --- a/docs/tasks/segment.md +++ b/docs/tasks/segment.md @@ -181,6 +181,6 @@ i.e. `yolo predict model=yolov8n-seg.onnx`. Usage examples are shown for your mo | [TF Edge TPU](https://coral.ai/docs/edgetpu/models-intro/) | `edgetpu` | `yolov8n-seg_edgetpu.tflite` | ✅ | `imgsz` | | [TF.js](https://www.tensorflow.org/js) | `tfjs` | `yolov8n-seg_web_model/` | ✅ | `imgsz` | | [PaddlePaddle](https://github.com/PaddlePaddle) | `paddle` | `yolov8n-seg_paddle_model/` | ✅ | `imgsz` | -| [NCNN](https://github.com/Tencent/ncnn) | `ncnn` | `yolov8n-seg_ncnn_model/` | ✅ | `imgsz`, `half` | +| [ncnn](https://github.com/Tencent/ncnn) | `ncnn` | `yolov8n-seg_ncnn_model/` | ✅ | `imgsz`, `half` | See full `export` details in the [Export](https://docs.ultralytics.com/modes/export/) page. \ No newline at end of file diff --git a/ultralytics/__init__.py b/ultralytics/__init__.py index 5800644..d337115 100644 --- a/ultralytics/__init__.py +++ b/ultralytics/__init__.py @@ -1,6 +1,6 @@ # Ultralytics YOLO 🚀, AGPL-3.0 license -__version__ = '8.0.131' +__version__ = '8.0.132' from ultralytics.hub import start from ultralytics.vit.rtdetr import RTDETR diff --git a/ultralytics/nn/autobackend.py b/ultralytics/nn/autobackend.py index e277957..566e276 100644 --- a/ultralytics/nn/autobackend.py +++ b/ultralytics/nn/autobackend.py @@ -3,6 +3,7 @@ import ast import contextlib import json +import os import platform import zipfile from collections import OrderedDict, namedtuple @@ -15,7 +16,7 @@ import torch import torch.nn as nn from PIL import Image -from ultralytics.yolo.utils import LINUX, LOGGER, ROOT, yaml_load +from ultralytics.yolo.utils import ARM64, LINUX, LOGGER, ROOT, yaml_load from ultralytics.yolo.utils.checks import check_requirements, check_suffix, check_version, check_yaml from ultralytics.yolo.utils.downloads import attempt_download_asset, is_url from ultralytics.yolo.utils.ops import xywh2xyxy @@ -75,6 +76,7 @@ class AutoBackend(nn.Module): | TensorFlow Lite | *.tflite | | TensorFlow Edge TPU | *_edgetpu.tflite | | PaddlePaddle | *_paddle_model | + | ncnn | *_ncnn_model | """ super().__init__() w = str(weights[0] if isinstance(weights, list) else weights) @@ -253,8 +255,19 @@ class AutoBackend(nn.Module): input_handle = predictor.get_input_handle(predictor.get_input_names()[0]) output_names = predictor.get_output_names() metadata = w.parents[1] / 'metadata.yaml' - elif ncnn: # PaddlePaddle - raise NotImplementedError('YOLOv8 NCNN inference is not currently supported.') + elif ncnn: # ncnn + LOGGER.info(f'Loading {w} for ncnn inference...') + check_requirements('git+https://github.com/Tencent/ncnn.git' if ARM64 else 'ncnn') # requires NCNN + import ncnn as pyncnn + net = pyncnn.Net() + net.opt.num_threads = os.cpu_count() + net.opt.use_vulkan_compute = cuda + w = Path(w) + if not w.is_file(): # if not *.param + w = next(w.glob('*.param')) # get *.param file from *_ncnn_model dir + net.load_param(str(w)) + net.load_model(str(w.with_suffix('.bin'))) + metadata = w.parent / 'metadata.yaml' elif triton: # NVIDIA Triton Inference Server LOGGER.info('Triton Inference Server not supported...') ''' @@ -358,6 +371,19 @@ class AutoBackend(nn.Module): self.input_handle.copy_from_cpu(im) self.predictor.run() y = [self.predictor.get_output_handle(x).copy_to_cpu() for x in self.output_names] + elif self.ncnn: # ncnn + im = (im[0] * 255.).cpu().numpy().astype(np.uint8) + im = np.ascontiguousarray(im.transpose(1, 2, 0)) + mat_in = self.pyncnn.Mat.from_pixels(im, self.pyncnn.Mat.PixelType.PIXEL_RGB, *im.shape[:2]) + mat_in.substract_mean_normalize([], [1 / 255.0, 1 / 255.0, 1 / 255.0]) + ex = self.net.create_extractor() + input_names, output_names = self.net.input_names(), self.net.output_names() + ex.input(input_names[0], mat_in) + y = [] + for output_name in output_names: + mat_out = self.pyncnn.Mat() + ex.extract(output_name, mat_out) + y.append(np.array(mat_out)[None]) elif self.triton: # NVIDIA Triton Inference Server y = self.model(im) else: # TensorFlow (SavedModel, GraphDef, Lite, Edge TPU) diff --git a/ultralytics/yolo/utils/benchmarks.py b/ultralytics/yolo/utils/benchmarks.py index f9e8f00..e84a1a6 100644 --- a/ultralytics/yolo/utils/benchmarks.py +++ b/ultralytics/yolo/utils/benchmarks.py @@ -21,7 +21,7 @@ TensorFlow Lite | `tflite` | yolov8n.tflite TensorFlow Edge TPU | `edgetpu` | yolov8n_edgetpu.tflite TensorFlow.js | `tfjs` | yolov8n_web_model/ PaddlePaddle | `paddle` | yolov8n_paddle_model/ -NCNN | `ncnn` | yolov8n_ncnn_model/ +ncnn | `ncnn` | yolov8n_ncnn_model/ """ import glob @@ -99,7 +99,7 @@ def benchmark(model=Path(SETTINGS['weights_dir']) / 'yolov8n.pt', # Predict assert model.task != 'pose' or i != 7, 'GraphDef Pose inference is not supported' - assert i not in (9, 10, 12), 'inference not supported' # Edge TPU, TF.js and NCNN are unsupported + assert i not in (9, 10), 'inference not supported' # Edge TPU and TF.js are unsupported assert i != 5 or platform.system() == 'Darwin', 'inference only supported on macOS>=10.13' # CoreML if not (ROOT / 'assets/bus.jpg').exists(): download(url='https://ultralytics.com/images/bus.jpg', dir=ROOT / 'assets')