From 598f17a4729e30da062c68a0592a20176b6dac85 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Sat, 31 Dec 2022 21:40:41 +0100 Subject: [PATCH] Add global `settings.yaml` in `USER_CONFIG_DIR` (#125) --- docs/index.md | 21 ++++- ultralytics/nn/autobackend.py | 3 +- ultralytics/nn/tasks.py | 3 +- ultralytics/yolo/configs/__init__.py | 16 ++-- ultralytics/yolo/data/base.py | 2 +- ultralytics/yolo/data/dataloaders/box.py | 0 ultralytics/yolo/data/dataloaders/segment.py | 0 ultralytics/yolo/data/dataloaders/v5loader.py | 2 +- ultralytics/yolo/data/utils.py | 4 +- ultralytics/yolo/engine/exporter.py | 6 +- ultralytics/yolo/engine/model.py | 3 +- ultralytics/yolo/engine/trainer.py | 4 +- ultralytics/yolo/utils/__init__.py | 87 +++++++++++++++++-- ultralytics/yolo/utils/files.py | 14 --- ultralytics/yolo/utils/instance.py | 4 +- ultralytics/yolo/v8/detect/val.py | 3 +- 16 files changed, 127 insertions(+), 45 deletions(-) delete mode 100644 ultralytics/yolo/data/dataloaders/box.py delete mode 100644 ultralytics/yolo/data/dataloaders/segment.py diff --git a/docs/index.md b/docs/index.md index 03f2776..91eb1a7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -37,4 +37,23 @@ For more information about the history and development of YOLO, you can refer to - Redmon, J., & Farhadi, A. (2015). You only look once: Unified, real-time object detection. In Proceedings of the IEEE conference on computer vision and pattern recognition (pp. 779-788). -- Redmon, J., & Farhadi, A. (2016). YOLO9000: Better, faster, stronger. In Proceedings \ No newline at end of file +- Redmon, J., & Farhadi, A. (2016). YOLO9000: Better, faster, stronger. In Proceedings + +### YOLOv8 by Ultralytics + +YOLOv8 is the latest version of the YOLO object detection and image segmentation model developed by +Ultralytics. YOLOv8 is a cutting-edge, state-of-the-art (SOTA) model that builds upon the success of previous YOLO +versions and introduces new features and improvements to further boost performance and flexibility. + +One key feature of YOLOv8 is its extensibility. It is designed as a framework that supports all previous versions of +YOLO, making it easy to switch between different versions and compare their performance. This makes YOLOv8 an ideal +choice for users who want to take advantage of the latest YOLO technology while still being able to use their existing +YOLO models. + +In addition to its extensibility, YOLOv8 includes a number of other innovations that make it an appealing choice for a +wide range of object detection and image segmentation tasks. These include a new backbone network, a new anchor-free +detection head, and a new loss function. YOLOv8 is also highly efficient and can be run on a variety of hardware +platforms, from CPUs to GPUs. + +Overall, YOLOv8 is a powerful and flexible tool for object detection and image segmentation that offers the best of both +worlds: the latest SOTA technology and the ability to use and compare all previous YOLO versions. \ No newline at end of file diff --git a/ultralytics/nn/autobackend.py b/ultralytics/nn/autobackend.py index 91da340..d4ad519 100644 --- a/ultralytics/nn/autobackend.py +++ b/ultralytics/nn/autobackend.py @@ -10,10 +10,9 @@ import torch import torch.nn as nn from PIL import Image -from ultralytics.yolo.utils import LOGGER, ROOT +from ultralytics.yolo.utils import LOGGER, ROOT, yaml_load from ultralytics.yolo.utils.checks import check_requirements, check_suffix, check_version from ultralytics.yolo.utils.downloads import attempt_download, is_url -from ultralytics.yolo.utils.files import yaml_load from ultralytics.yolo.utils.ops import xywh2xyxy diff --git a/ultralytics/nn/tasks.py b/ultralytics/nn/tasks.py index facae8b..e58aa93 100644 --- a/ultralytics/nn/tasks.py +++ b/ultralytics/nn/tasks.py @@ -10,9 +10,8 @@ import torchvision from ultralytics.nn.modules import (C1, C2, C3, C3TR, SPP, SPPF, Bottleneck, BottleneckCSP, C2f, C3Ghost, C3x, Classify, Concat, Conv, ConvTranspose, Detect, DWConv, DWConvTranspose2d, Ensemble, Focus, GhostBottleneck, GhostConv, Segment) -from ultralytics.yolo.utils import LOGGER, colorstr +from ultralytics.yolo.utils import LOGGER, colorstr, yaml_load from ultralytics.yolo.utils.checks import check_yaml -from ultralytics.yolo.utils.files import yaml_load from ultralytics.yolo.utils.torch_utils import (fuse_conv_and_bn, initialize_weights, intersect_state_dicts, make_divisible, model_info, scale_img, time_sync) diff --git a/ultralytics/yolo/configs/__init__.py b/ultralytics/yolo/configs/__init__.py index 7786668..f0bb8ec 100644 --- a/ultralytics/yolo/configs/__init__.py +++ b/ultralytics/yolo/configs/__init__.py @@ -6,13 +6,19 @@ from omegaconf import DictConfig, OmegaConf from ultralytics.yolo.configs.hydra_patch import check_config_mismatch -def get_config(config: Union[str, DictConfig], overrides: Union[str, Dict] = {}): +def get_config(config: Union[str, DictConfig], overrides: Union[str, Dict] = None): """ - Accepts yaml file name or DictConfig containing experiment configuration. - Returns training args namespace - :param overrides: Overrides str or Dict - :param config: Optional file name or DictConfig object + Load and merge configuration data from a file or dictionary. + + Args: + config (Union[str, DictConfig]): Configuration data in the form of a file name or a DictConfig object. + overrides (Union[str, Dict], optional): Overrides in the form of a file name or a dictionary. Default is None. + + Returns: + OmegaConf.Namespace: Training arguments namespace. """ + if overrides is None: + overrides = {} if isinstance(config, (str, Path)): config = OmegaConf.load(config) elif isinstance(config, Dict): diff --git a/ultralytics/yolo/data/base.py b/ultralytics/yolo/data/base.py index d6ceff7..e835441 100644 --- a/ultralytics/yolo/data/base.py +++ b/ultralytics/yolo/data/base.py @@ -91,7 +91,7 @@ class BaseDataset(Dataset): # self.img_files = sorted([x for x in f if x.suffix[1:].lower() in IMG_FORMATS]) # pathlib assert im_files, f"{self.prefix}No images found" except Exception as e: - raise Exception(f"{self.prefix}Error loading data from {img_path}: {e}\n{HELP_URL}") + raise FileNotFoundError(f"{self.prefix}Error loading data from {img_path}: {e}\n{HELP_URL}") from e return im_files def update_labels(self, include_class: Optional[list]): diff --git a/ultralytics/yolo/data/dataloaders/box.py b/ultralytics/yolo/data/dataloaders/box.py deleted file mode 100644 index e69de29..0000000 diff --git a/ultralytics/yolo/data/dataloaders/segment.py b/ultralytics/yolo/data/dataloaders/segment.py deleted file mode 100644 index e69de29..0000000 diff --git a/ultralytics/yolo/data/dataloaders/v5loader.py b/ultralytics/yolo/data/dataloaders/v5loader.py index 6bbed71..9dee14b 100644 --- a/ultralytics/yolo/data/dataloaders/v5loader.py +++ b/ultralytics/yolo/data/dataloaders/v5loader.py @@ -484,7 +484,7 @@ class LoadImagesAndLabels(Dataset): # self.img_files = sorted([x for x in f if x.suffix[1:].lower() in IMG_FORMATS]) # pathlib assert self.im_files, f'{prefix}No images found' except Exception as e: - raise Exception(f'{prefix}Error loading data from {path}: {e}\n{HELP_URL}') from e + raise FileNotFoundError(f'{prefix}Error loading data from {path}: {e}\n{HELP_URL}') from e # Check cache self.label_files = img2label_paths(self.im_files) # labels diff --git a/ultralytics/yolo/data/utils.py b/ultralytics/yolo/data/utils.py index 1800054..25554e9 100644 --- a/ultralytics/yolo/data/utils.py +++ b/ultralytics/yolo/data/utils.py @@ -12,10 +12,10 @@ import numpy as np import torch from PIL import ExifTags, Image, ImageOps -from ultralytics.yolo.utils import LOGGER, ROOT, colorstr +from ultralytics.yolo.utils import LOGGER, ROOT, colorstr, yaml_load from ultralytics.yolo.utils.checks import check_file, check_font, is_ascii from ultralytics.yolo.utils.downloads import download -from ultralytics.yolo.utils.files import unzip_file, yaml_load +from ultralytics.yolo.utils.files import unzip_file from ..utils.ops import segments2boxes diff --git a/ultralytics/yolo/engine/exporter.py b/ultralytics/yolo/engine/exporter.py index db480f3..0f2fab2 100644 --- a/ultralytics/yolo/engine/exporter.py +++ b/ultralytics/yolo/engine/exporter.py @@ -70,9 +70,9 @@ from ultralytics.nn.tasks import ClassificationModel, DetectionModel, Segmentati from ultralytics.yolo.configs import get_config from ultralytics.yolo.data.dataloaders.stream_loaders import LoadImages from ultralytics.yolo.data.utils import check_dataset -from ultralytics.yolo.utils import DEFAULT_CONFIG, LOGGER, colorstr, get_default_args +from ultralytics.yolo.utils import DEFAULT_CONFIG, LOGGER, colorstr, get_default_args, yaml_save from ultralytics.yolo.utils.checks import check_imgsz, check_requirements, check_version, check_yaml -from ultralytics.yolo.utils.files import file_size, increment_path, yaml_save +from ultralytics.yolo.utils.files import file_size, increment_path from ultralytics.yolo.utils.ops import Profile from ultralytics.yolo.utils.torch_utils import guess_task_from_head, select_device, smart_inference_mode @@ -198,7 +198,7 @@ class Exporter: self.im = im self.model = model self.file = file - self.output_shape = tuple(y.shape) + self.output_shape = tuple(y.shape) if isinstance(y, torch.Tensor) else (x.shape for x in y) self.metadata = {'stride': int(max(model.stride)), 'names': model.names} # model metadata self.pretty_name = self.file.stem.replace('yolo', 'YOLO') diff --git a/ultralytics/yolo/engine/model.py b/ultralytics/yolo/engine/model.py index 0735580..da53a62 100644 --- a/ultralytics/yolo/engine/model.py +++ b/ultralytics/yolo/engine/model.py @@ -4,9 +4,8 @@ from ultralytics import yolo # noqa from ultralytics.nn.tasks import ClassificationModel, DetectionModel, SegmentationModel, attempt_load_weights from ultralytics.yolo.configs import get_config from ultralytics.yolo.engine.exporter import Exporter -from ultralytics.yolo.utils import DEFAULT_CONFIG, HELP_MSG, LOGGER +from ultralytics.yolo.utils import DEFAULT_CONFIG, HELP_MSG, LOGGER, yaml_load from ultralytics.yolo.utils.checks import check_imgsz, check_yaml -from ultralytics.yolo.utils.files import yaml_load from ultralytics.yolo.utils.torch_utils import guess_task_from_head, smart_inference_mode # Map head to model, trainer, validator, and predictor classes diff --git a/ultralytics/yolo/engine/trainer.py b/ultralytics/yolo/engine/trainer.py index 7fb1c24..cdb042e 100644 --- a/ultralytics/yolo/engine/trainer.py +++ b/ultralytics/yolo/engine/trainer.py @@ -25,10 +25,10 @@ import ultralytics.yolo.utils.callbacks as callbacks from ultralytics import __version__ from ultralytics.yolo.configs import get_config from ultralytics.yolo.data.utils import check_dataset, check_dataset_yaml -from ultralytics.yolo.utils import DEFAULT_CONFIG, LOGGER, RANK, TQDM_BAR_FORMAT, colorstr +from ultralytics.yolo.utils import DEFAULT_CONFIG, LOGGER, RANK, TQDM_BAR_FORMAT, colorstr, yaml_save from ultralytics.yolo.utils.checks import check_file, print_args from ultralytics.yolo.utils.dist import ddp_cleanup, generate_ddp_command -from ultralytics.yolo.utils.files import get_latest_run, increment_path, yaml_save +from ultralytics.yolo.utils.files import get_latest_run, increment_path from ultralytics.yolo.utils.torch_utils import ModelEMA, de_parallel, init_seeds, one_cycle, strip_optimizer diff --git a/ultralytics/yolo/utils/__init__.py b/ultralytics/yolo/utils/__init__.py index 81ff33c..fb6bb36 100644 --- a/ultralytics/yolo/utils/__init__.py +++ b/ultralytics/yolo/utils/__init__.py @@ -10,6 +10,7 @@ from pathlib import Path import cv2 import pandas as pd +import yaml # Constants FILE = Path(__file__).resolve() @@ -224,13 +225,6 @@ def set_logging(name=LOGGING_NAME, verbose=True): "propagate": False,}}}) -set_logging(LOGGING_NAME) # run before defining LOGGER -LOGGER = logging.getLogger(LOGGING_NAME) # define globally (used in train.py, val.py, detect.py, etc.) -if platform.system() == 'Windows': - for fn in LOGGER.info, LOGGER.warning: - setattr(LOGGER, fn.__name__, lambda x: fn(emojis(x))) # emoji safe logging - - class TryExcept(contextlib.ContextDecorator): # YOLOv5 TryExcept class. Usage: @TryExcept() decorator or 'with TryExcept():' context manager def __init__(self, msg=''): @@ -253,3 +247,82 @@ def threaded(func): return thread return wrapper + + +def get_settings(file=USER_CONFIG_DIR / 'settings.yaml'): + """ + Function that loads a global settings YAML, or creates it and populates it with default values if it does not exist. + + If the datasets or weights directories are set to None, the current working directory will be used. + The 'sync' setting determines whether analytics will be synced to help with YOLO development. + """ + from ultralytics.yolo.utils.torch_utils import torch_distributed_zero_first + + with torch_distributed_zero_first(RANK): + if not file.exists(): + settings = { + 'datasets_dir': None, # default datasets directory. If None, current working directory is used. + 'weights_dir': None, # default weights directory. If None, current working directory is used. + 'sync': True} # sync analytics to help with YOLO development + yaml_save(file, settings) + + return yaml_load(file) + + +def yaml_save(file='data.yaml', data=None): + """ + Save YAML data to a file. + + Args: + file (str, optional): File name. Default is 'data.yaml'. + data (dict, optional): Data to save in YAML format. Default is None. + + Returns: + None: Data is saved to the specified file. + """ + file = Path(file) + if not file.parent.exists(): + # Create parent directories if they don't exist + file.parent.mkdir(parents=True, exist_ok=True) + + with open(file, 'w') as f: + # Dump data to file in YAML format, converting Path objects to strings + yaml.safe_dump({k: str(v) if isinstance(v, Path) else v for k, v in data.items()}, f, sort_keys=False) + + +def yaml_load(file='data.yaml'): + """ + Load YAML data from a file. + + Args: + file (str, optional): File name. Default is 'data.yaml'. + + Returns: + dict: YAML data and file name. + """ + with open(file, errors='ignore') as f: + # Add YAML filename to dict and return + return {**yaml.safe_load(f), 'yaml_file': file} + + +# Run below code on utils init ----------------------------------------------------------------------------------------- + +# Set logger +set_logging(LOGGING_NAME) # run before defining LOGGER +LOGGER = logging.getLogger(LOGGING_NAME) # define globally (used in train.py, val.py, detect.py, etc.) +if platform.system() == 'Windows': + for fn in LOGGER.info, LOGGER.warning: + setattr(LOGGER, fn.__name__, lambda x: fn(emojis(x))) # emoji safe logging + +# Check first-install steps +SETTINGS = get_settings() + + +def set_settings(kwargs, file=USER_CONFIG_DIR / 'settings.yaml'): + """ + Function that runs on a first-time ultralytics package installation to set up global settings and create necessary + directories. + """ + SETTINGS.update(kwargs) + + yaml_save(file, SETTINGS) diff --git a/ultralytics/yolo/utils/files.py b/ultralytics/yolo/utils/files.py index e185226..96c6367 100644 --- a/ultralytics/yolo/utils/files.py +++ b/ultralytics/yolo/utils/files.py @@ -6,8 +6,6 @@ from datetime import datetime from pathlib import Path from zipfile import ZipFile -import yaml - class WorkingDirectory(contextlib.ContextDecorator): # Usage: @WorkingDirectory(dir) decorator or 'with WorkingDirectory(dir):' context manager @@ -57,18 +55,6 @@ def increment_path(path, exist_ok=False, sep='', mkdir=False): return path -def yaml_save(file='data.yaml', data=None): - # Single-line safe yaml saving - with open(file, 'w') as f: - yaml.safe_dump({k: str(v) if isinstance(v, Path) else v for k, v in data.items()}, f, sort_keys=False) - - -def yaml_load(file='data.yaml'): - # Single-line safe yaml loading - with open(file, errors='ignore') as f: - return {**yaml.safe_load(f), 'yaml_file': file} # add YAML filename to dict and return - - def unzip_file(file, path=None, exclude=('.DS_Store', '__MACOSX')): # Unzip a *.zip file to path/, excluding files containing strings in exclude list if path is None: diff --git a/ultralytics/yolo/utils/instance.py b/ultralytics/yolo/utils/instance.py index bc418bc..d2873fd 100644 --- a/ultralytics/yolo/utils/instance.py +++ b/ultralytics/yolo/utils/instance.py @@ -162,13 +162,15 @@ class Bboxes: class Instances: - def __init__(self, bboxes, segments=[], keypoints=None, bbox_format="xywh", normalized=True) -> None: + def __init__(self, bboxes, segments=None, keypoints=None, bbox_format="xywh", normalized=True) -> None: """ Args: bboxes (ndarray): bboxes with shape [N, 4]. segments (list | ndarray): segments. keypoints (ndarray): keypoints with shape [N, 17, 2]. """ + if segments is None: + segments = [] self._bboxes = Bboxes(bboxes=bboxes, format=bbox_format) self.keypoints = keypoints self.normalized = normalized diff --git a/ultralytics/yolo/v8/detect/val.py b/ultralytics/yolo/v8/detect/val.py index e45e9a6..c14c097 100644 --- a/ultralytics/yolo/v8/detect/val.py +++ b/ultralytics/yolo/v8/detect/val.py @@ -8,9 +8,8 @@ import torch from ultralytics.yolo.data import build_dataloader from ultralytics.yolo.data.dataloaders.v5loader import create_dataloader from ultralytics.yolo.engine.validator import BaseValidator -from ultralytics.yolo.utils import DEFAULT_CONFIG, colorstr, ops +from ultralytics.yolo.utils import DEFAULT_CONFIG, colorstr, ops, yaml_load from ultralytics.yolo.utils.checks import check_file, check_requirements -from ultralytics.yolo.utils.files import yaml_load from ultralytics.yolo.utils.metrics import ConfusionMatrix, DetMetrics, box_iou from ultralytics.yolo.utils.plotting import output_to_target, plot_images from ultralytics.yolo.utils.torch_utils import de_parallel