From 965e40595709857ea86a3bff40d3c98091e36b33 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sat, 22 Jul 2023 19:44:43 +0200 Subject: [PATCH] `ultralytics 8.0.140` improved robustness to model path spaces (#3879) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Lukas Hennies <45569834+Gornoka@users.noreply.github.com> --- .github/workflows/ci.yaml | 10 ++-- .gitignore | 4 +- CONTRIBUTING.md | 2 +- tests/test_python.py | 2 +- ultralytics/__init__.py | 2 +- ultralytics/engine/exporter.py | 31 +++++++------ ultralytics/utils/callbacks/tensorboard.py | 4 +- ultralytics/utils/downloads.py | 2 +- ultralytics/utils/files.py | 53 ++++++++++++++++++++++ 9 files changed, 85 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f70d352..c358334 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -112,22 +112,22 @@ jobs: shell: python run: | from ultralytics.utils.benchmarks import benchmark - benchmark(model='${{ matrix.model }}.pt', imgsz=160, half=False, hard_fail=0.26) + benchmark(model='path with spaces/${{ matrix.model }}.pt', imgsz=160, half=False, hard_fail=0.26) - name: Benchmark SegmentationModel shell: python run: | from ultralytics.utils.benchmarks import benchmark - benchmark(model='${{ matrix.model }}-seg.pt', imgsz=160, half=False, hard_fail=0.30) + benchmark(model='path with spaces/${{ matrix.model }}-seg.pt', imgsz=160, half=False, hard_fail=0.30) - name: Benchmark ClassificationModel shell: python run: | from ultralytics.utils.benchmarks import benchmark - benchmark(model='${{ matrix.model }}-cls.pt', imgsz=160, half=False, hard_fail=0.36) + benchmark(model='path with spaces/${{ matrix.model }}-cls.pt', imgsz=160, half=False, hard_fail=0.36) - name: Benchmark PoseModel shell: python run: | from ultralytics.utils.benchmarks import benchmark - benchmark(model='${{ matrix.model }}-pose.pt', imgsz=160, half=False, hard_fail=0.17) + benchmark(model='path with spaces/${{ matrix.model }}-pose.pt', imgsz=160, half=False, hard_fail=0.17) - name: Benchmark Summary run: | cat benchmarks.log @@ -141,12 +141,10 @@ jobs: matrix: os: [ubuntu-latest] python-version: ['3.11'] - model: [yolov8n] torch: [latest] include: - os: ubuntu-latest python-version: '3.8' # torch 1.7.0 requires python >=3.6, <=3.8 - model: yolov8n torch: '1.8.0' # min torch version CI https://pypi.org/project/torchvision/ steps: - uses: actions/checkout@v3 diff --git a/.gitignore b/.gitignore index f8dcff9..069a941 100644 --- a/.gitignore +++ b/.gitignore @@ -136,7 +136,6 @@ dmypy.json datasets/ runs/ wandb/ - .DS_Store # Neural Network weights ----------------------------------------------------------------------------------------------- @@ -154,3 +153,6 @@ weights/ *_web_model/ *_openvino_model/ *_paddle_model/ + +# Autogenerated files for tests +/ultralytics/assets/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9326906..a4f12c6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -65,7 +65,7 @@ Here is an example: ```python """ - What the function does. Performs NMS on given detection predictions. + What the function does. Performs NMS on given detection predictions. Args: arg1: The description of the 1st argument diff --git a/tests/test_python.py b/tests/test_python.py index 6502cf5..4053480 100644 --- a/tests/test_python.py +++ b/tests/test_python.py @@ -12,7 +12,7 @@ from ultralytics import RTDETR, YOLO from ultralytics.data.build import load_inference_source from ultralytics.utils import LINUX, ONLINE, ROOT, SETTINGS -MODEL = Path(SETTINGS['weights_dir']) / 'yolov8n.pt' +MODEL = Path(SETTINGS['weights_dir']) / 'path with spaces' / 'yolov8n.pt' # test spaces in path CFG = 'yolov8n.yaml' SOURCE = ROOT / 'assets/bus.jpg' SOURCE_GREYSCALE = Path(f'{SOURCE.parent / SOURCE.stem}_greyscale.jpg') diff --git a/ultralytics/__init__.py b/ultralytics/__init__.py index b387a07..b512491 100644 --- a/ultralytics/__init__.py +++ b/ultralytics/__init__.py @@ -1,6 +1,6 @@ # Ultralytics YOLO 🚀, AGPL-3.0 license -__version__ = '8.0.139' +__version__ = '8.0.140' from ultralytics.engine.model import YOLO from ultralytics.hub import start diff --git a/ultralytics/engine/exporter.py b/ultralytics/engine/exporter.py index 7811e1b..1793028 100644 --- a/ultralytics/engine/exporter.py +++ b/ultralytics/engine/exporter.py @@ -68,7 +68,7 @@ from ultralytics.utils import (ARM64, DEFAULT_CFG, LINUX, LOGGER, MACOS, ROOT, W colorstr, get_default_args, yaml_save) from ultralytics.utils.checks import check_imgsz, check_requirements, check_version from ultralytics.utils.downloads import attempt_download_asset, get_github_assets -from ultralytics.utils.files import file_size +from ultralytics.utils.files import file_size, spaces_in_path from ultralytics.utils.ops import Profile from ultralytics.utils.torch_utils import get_latest_opset, select_device, smart_inference_mode @@ -112,7 +112,7 @@ def try_export(inner_func): try: with Profile() as dt: f, model = inner_func(*args, **kwargs) - LOGGER.info(f'{prefix} export success ✅ {dt.t:.1f}s, saved as {f} ({file_size(f):.1f} MB)') + LOGGER.info(f"{prefix} export success ✅ {dt.t:.1f}s, saved as '{f}' ({file_size(f):.1f} MB)") return f, model except Exception as e: LOGGER.info(f'{prefix} export failure ❌ {dt.t:.1f}s: {e}') @@ -230,7 +230,7 @@ class Exporter: if model.task == 'pose': self.metadata['kpt_shape'] = model.model[-1].kpt_shape - LOGGER.info(f"\n{colorstr('PyTorch:')} starting from {file} with input shape {tuple(im.shape)} BCHW and " + LOGGER.info(f"\n{colorstr('PyTorch:')} starting from '{file}' with input shape {tuple(im.shape)} BCHW and " f'output shape(s) {self.output_shape} ({file_size(file):.1f} MB)') # Exports @@ -340,7 +340,7 @@ class Exporter: import onnxsim LOGGER.info(f'{prefix} simplifying with onnxsim {onnxsim.__version__}...') - # subprocess.run(f'onnxsim {f} {f}', shell=True) + # subprocess.run(f'onnxsim "{f}" "{f}"', shell=True) model_onnx, check = onnxsim.simplify(model_onnx) assert check, 'Simplified ONNX model could not be validated' except Exception as e: @@ -410,7 +410,7 @@ class Exporter: LOGGER.info(f'\n{prefix} starting export with ncnn {ncnn.__version__}...') f = Path(str(self.file).replace(self.file.suffix, f'_ncnn_model{os.sep}')) - f_ts = str(self.file.with_suffix('.torchscript')) + f_ts = self.file.with_suffix('.torchscript') pnnx_filename = 'pnnx.exe' if WINDOWS else 'pnnx' if Path(pnnx_filename).is_file(): @@ -434,7 +434,7 @@ class Exporter: cmd = [ str(pnnx), - f_ts, + str(f_ts), f'pnnxparam={f / "model.pnnx.param"}', f'pnnxbin={f / "model.pnnx.bin"}', f'pnnxpy={f / "model_pnnx.py"}', @@ -586,8 +586,8 @@ class Exporter: # Export to TF int8 = '-oiqt -qt per-tensor' if self.args.int8 else '' - cmd = f'onnx2tf -i {f_onnx} -o {f} -nuo --non_verbose {int8}' - LOGGER.info(f"\n{prefix} running '{cmd.strip()}'") + cmd = f'onnx2tf -i "{f_onnx}" -o "{f}" -nuo --non_verbose {int8}' + LOGGER.info(f"\n{prefix} running '{cmd}'") subprocess.run(cmd, shell=True) yaml_save(f / 'metadata.yaml', self.metadata) # add metadata.yaml @@ -659,9 +659,9 @@ class Exporter: LOGGER.info(f'\n{prefix} starting export with Edge TPU compiler {ver}...') f = str(tflite_model).replace('.tflite', '_edgetpu.tflite') # Edge TPU model - cmd = f'edgetpu_compiler -s -d -k 10 --out_dir {Path(f).parent} {tflite_model}' + cmd = f'edgetpu_compiler -s -d -k 10 --out_dir "{Path(f).parent}" "{tflite_model}"' LOGGER.info(f"{prefix} running '{cmd}'") - subprocess.run(cmd.split(), check=True) + subprocess.run(cmd, shell=True) self._add_tflite_metadata(f) return f, None @@ -674,7 +674,7 @@ class Exporter: LOGGER.info(f'\n{prefix} starting export with tensorflowjs {tfjs.__version__}...') f = str(self.file).replace(self.file.suffix, '_web_model') # js dir - f_pb = self.file.with_suffix('.pb') # *.pb path + f_pb = str(self.file.with_suffix('.pb')) # *.pb path gd = tf.Graph().as_graph_def() # TF GraphDef with open(f_pb, 'rb') as file: @@ -682,8 +682,13 @@ class Exporter: outputs = ','.join(gd_outputs(gd)) LOGGER.info(f'\n{prefix} output node names: {outputs}') - cmd = f'tensorflowjs_converter --input_format=tf_frozen_model --output_node_names={outputs} {f_pb} {f}' - subprocess.run(cmd.split(), check=True) + with spaces_in_path(f_pb) as fpb_, spaces_in_path(f) as f_: # exporter can not handle spaces in path + cmd = f'tensorflowjs_converter --input_format=tf_frozen_model --output_node_names={outputs} "{fpb_}" "{f_}"' + LOGGER.info(f"{prefix} running '{cmd}'") + subprocess.run(cmd, shell=True) + + if ' ' in str(f): + LOGGER.warning(f"{prefix} WARNING ⚠️ your model may not work correctly with spaces in path '{f}'.") # f_json = Path(f) / 'model.json' # *.json path # with open(f_json, 'w') as j: # sort JSON Identity_* in ascending order diff --git a/ultralytics/utils/callbacks/tensorboard.py b/ultralytics/utils/callbacks/tensorboard.py index 7ddf1bb..bd2300b 100644 --- a/ultralytics/utils/callbacks/tensorboard.py +++ b/ultralytics/utils/callbacks/tensorboard.py @@ -6,7 +6,9 @@ try: from torch.utils.tensorboard import SummaryWriter assert not TESTS_RUNNING # do not log pytest -except (ImportError, AssertionError): + +# TypeError for handling 'Descriptors cannot not be created directly.' protobuf errors in Windows +except (ImportError, AssertionError, TypeError): SummaryWriter = None writer = None # TensorBoard SummaryWriter instance diff --git a/ultralytics/utils/downloads.py b/ultralytics/utils/downloads.py index 0958f35..69ec7b8 100644 --- a/ultralytics/utils/downloads.py +++ b/ultralytics/utils/downloads.py @@ -149,7 +149,7 @@ def safe_download(url, elif not f.is_file(): # URL and file do not exist assert dir or file, 'dir or file required for download' f = dir / url2file(url) if dir else Path(file) - desc = f'Downloading {clean_url(url)} to {f}' + desc = f"Downloading {clean_url(url)} to '{f}'" LOGGER.info(f'{desc}...') f.parent.mkdir(parents=True, exist_ok=True) # make directory if missing check_disk_space(url) diff --git a/ultralytics/utils/files.py b/ultralytics/utils/files.py index 6359947..90c8d28 100644 --- a/ultralytics/utils/files.py +++ b/ultralytics/utils/files.py @@ -4,6 +4,8 @@ import contextlib import glob import os import shutil +import tempfile +from contextlib import contextmanager from datetime import datetime from pathlib import Path @@ -25,6 +27,57 @@ class WorkingDirectory(contextlib.ContextDecorator): os.chdir(self.cwd) +@contextmanager +def spaces_in_path(path): + """ + Context manager to handle paths with spaces in their names. + If a path contains spaces, it replaces them with underscores, copies the file/directory to the new path, + executes the context code block, then copies the file/directory back to its original location. + + Args: + path (str | Path): The original path. + + Yields: + Path: Temporary path with spaces replaced by underscores if spaces were present, otherwise the original path. + + Examples: + with spaces_in_path('/path/with spaces') as new_path: + # your code here + """ + + # If path has spaces, replace them with underscores + if ' ' in str(path): + string = isinstance(path, str) # input type + path = Path(path) + + # Create a temporary directory and construct the new path + with tempfile.TemporaryDirectory() as tmp_dir: + tmp_path = Path(tmp_dir) / path.name.replace(' ', '_') + + # Copy file/directory + if path.is_dir(): + # tmp_path.mkdir(parents=True, exist_ok=True) + shutil.copytree(path, tmp_path) + elif path.is_file(): + tmp_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, tmp_path) + + try: + # Yield the temporary path + yield str(tmp_path) if string else tmp_path + + finally: + # Copy file/directory back + if tmp_path.is_dir(): + shutil.copytree(tmp_path, path, dirs_exist_ok=True) + elif tmp_path.is_file(): + shutil.copy2(tmp_path, path) # Copy back the file + + else: + # If there are no spaces, just yield the original path + yield path + + def increment_path(path, exist_ok=False, sep='', mkdir=False): """ Increments a file or directory path, i.e. runs/exp --> runs/exp{sep}2, runs/exp{sep}3, ... etc.