ultralytics 8.0.46 TFLite and Benchmarks updates (#1141)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

ultralytics/yolo/utils/__init__.py

@@ -354,22 +354,6 @@ def get_git_branch():
     return None  # if not git dir or on error
 
 
-def get_latest_pypi_version(package_name='ultralytics'):
-    """
-    Returns the latest version of a PyPI package without downloading or installing it.
-
-    Parameters:
-        package_name (str): The name of the package to find the latest version for.
-
-    Returns:
-        str: The latest version of the package.
-    """
-    response = requests.get(f'https://pypi.org/pypi/{package_name}/json')
-    if response.status_code == 200:
-        return response.json()['info']['version']
-    return None
-
-
 def get_default_args(func):
     """Returns a dictionary of default arguments for a function.
 
@@ -611,7 +595,7 @@ def set_settings(kwargs, file=USER_CONFIG_DIR / 'settings.yaml'):
 # Run below code on yolo/utils init ------------------------------------------------------------------------------------
 
 # Set logger
-set_logging(LOGGING_NAME)  # run before defining LOGGER
+set_logging(LOGGING_NAME, verbose=VERBOSE)  # run before defining LOGGER
 LOGGER = logging.getLogger(LOGGING_NAME)  # define globally (used in train.py, val.py, detect.py, etc.)
 if WINDOWS:
     for fn in LOGGER.info, LOGGER.warning:

ultralytics/yolo/utils/benchmarks.py

@@ -37,11 +37,7 @@ from ultralytics.yolo.utils.files import file_size
 from ultralytics.yolo.utils.torch_utils import select_device
 
 
-def run_benchmarks(model=Path(SETTINGS['weights_dir']) / 'yolov8n.pt',
-                   imgsz=640,
-                   half=False,
-                   device='cpu',
-                   hard_fail=False):
+def benchmark(model=Path(SETTINGS['weights_dir']) / 'yolov8n.pt', imgsz=160, half=False, device='cpu', hard_fail=0.30):
     device = select_device(device, verbose=False)
     if isinstance(model, (str, Path)):
         model = YOLO(model)
@@ -52,6 +48,7 @@ def run_benchmarks(model=Path(SETTINGS['weights_dir']) / 'yolov8n.pt',
         try:
             assert i not in (9, 10), 'inference not supported'  # Edge TPU and TF.js are unsupported
             assert i != 5 or platform.system() == 'Darwin', 'inference only supported on macOS>=10.13'  # CoreML
+            assert i != 11 or model.task != 'classify', 'paddle-classify bug'
 
             if 'cpu' in device.type:
                 assert cpu, 'inference not supported on CPU'
@@ -85,26 +82,28 @@ def run_benchmarks(model=Path(SETTINGS['weights_dir']) / 'yolov8n.pt',
             y.append([name, '✅', round(file_size(filename), 1), round(metric, 4), round(speed, 2)])
         except Exception as e:
             if hard_fail:
-                assert type(e) is AssertionError, f'Benchmark --hard-fail for {name}: {e}'
+                assert type(e) is AssertionError, f'Benchmark hard_fail for {name}: {e}'
             LOGGER.warning(f'ERROR ❌️ Benchmark failure for {name}: {e}')
             y.append([name, '❌', None, None, None])  # mAP, t_inference
 
     # Print results
-    LOGGER.info('\n')
     check_yolo(device=device)  # print system info
-    c = ['Format', 'Status❔', 'Size (MB)', key, 'Inference time (ms/im)'] if map else ['Format', 'Export', '', '']
+    c = ['Format', 'Status❔', 'Size (MB)', key, 'Inference time (ms/im)']
     df = pd.DataFrame(y, columns=c)
-    LOGGER.info(f'\nBenchmarks complete for {Path(model.ckpt_path).name} on {data} at imgsz={imgsz} '
-                f'({time.time() - t0:.2f}s)')
-    LOGGER.info(str(df if map else df.iloc[:, :2]))
 
-    if hard_fail and isinstance(hard_fail, str):
+    name = Path(model.ckpt_path).name
+    s = f'\nBenchmarks complete for {name} on {data} at imgsz={imgsz} ({time.time() - t0:.2f}s)\n{df}\n'
+    LOGGER.info(s)
+    with open('benchmarks.log', 'a') as f:
+        f.write(s)
+
+    if hard_fail and isinstance(hard_fail, float):
         metrics = df[key].array  # values to compare to floor
-        floor = eval(hard_fail)  # minimum metric floor to pass, i.e. = 0.29 mAP for YOLOv5n
-        assert all(x > floor for x in metrics if pd.notna(x)), f'HARD FAIL: metric < floor {floor}'
+        floor = hard_fail  # minimum metric floor to pass, i.e. = 0.29 mAP for YOLOv5n
+        assert all(x > floor for x in metrics if pd.notna(x)), f'HARD FAIL: one or more metric(s) < floor {floor}'
 
     return df
 
 
 if __name__ == '__main__':
-    run_benchmarks()
+    benchmark()

ultralytics/yolo/utils/checks.py

@@ -16,6 +16,7 @@ import cv2
 import numpy as np
 import pkg_resources as pkg
 import psutil
+import requests
 import torch
 from matplotlib import font_manager
 
@@ -117,6 +118,31 @@ def check_version(current: str = '0.0.0',
     return result
 
 
+def check_latest_pypi_version(package_name='ultralytics'):
+    """
+    Returns the latest version of a PyPI package without downloading or installing it.
+
+    Parameters:
+        package_name (str): The name of the package to find the latest version for.
+
+    Returns:
+        str: The latest version of the package.
+    """
+    response = requests.get(f'https://pypi.org/pypi/{package_name}/json')
+    if response.status_code == 200:
+        return response.json()['info']['version']
+    return None
+
+
+def check_pip_update():
+    from ultralytics import __version__
+    latest = check_latest_pypi_version()
+    latest = '9.0.0'
+    if pkg.parse_version(__version__) < pkg.parse_version(latest):
+        LOGGER.info(f'New https://pypi.org/project/ultralytics/{latest} available 😃 '
+                    f"Update with 'pip install -U ultralytics'")
+
+
 def check_font(font='Arial.ttf'):
     """
     Find font locally or download to user's configuration directory if it does not already exist.

ultralytics/yolo/utils/dist.py

@@ -1,10 +1,12 @@
 # Ultralytics YOLO 🚀, GPL-3.0 license
 
 import os
+import re
 import shutil
 import socket
 import sys
 import tempfile
+from pathlib import Path
 
 from . import USER_CONFIG_DIR
 from .torch_utils import TORCH_1_9
@@ -22,12 +24,12 @@ def find_free_network_port() -> int:
 
 
 def generate_ddp_file(trainer):
-    import_path = '.'.join(str(trainer.__class__).split('.')[1:-1])
+    module, name = f'{trainer.__class__.__module__}.{trainer.__class__.__name__}'.rsplit('.', 1)
 
     content = f'''cfg = {vars(trainer.args)} \nif __name__ == "__main__":
-    from ultralytics.{import_path} import {trainer.__class__.__name__}
+    from {module} import {name}
 
-    trainer = {trainer.__class__.__name__}(cfg=cfg)
+    trainer = {name}(cfg=cfg)
     trainer.train()'''
     (USER_CONFIG_DIR / 'DDP').mkdir(exist_ok=True)
     with tempfile.NamedTemporaryFile(prefix='_temp_',
@@ -41,12 +43,12 @@ def generate_ddp_file(trainer):
 
 
 def generate_ddp_command(world_size, trainer):
-    import __main__  # local import to avoid https://github.com/Lightning-AI/lightning/issues/15218
-    file = os.path.abspath(sys.argv[0])
-    using_cli = not file.endswith('.py')
+    import __main__  # noqa local import to avoid https://github.com/Lightning-AI/lightning/issues/15218
     if not trainer.resume:
         shutil.rmtree(trainer.save_dir)  # remove the save_dir
-    if using_cli:
+    file = str(Path(sys.argv[0]).resolve())
+    safe_pattern = re.compile(r'^[a-zA-Z0-9_. /\\-]{1,128}$')  # allowed characters and maximum of 100 characters
+    if not (safe_pattern.match(file) and Path(file).exists() and file.endswith('.py')):  # using CLI
         file = generate_ddp_file(trainer)
     dist_cmd = 'torch.distributed.run' if TORCH_1_9 else 'torch.distributed.launch'
     port = find_free_network_port()