From af6e3c536be860aabf9cfca5ddb2ff960376edf6 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Mon, 2 Jan 2023 15:55:32 +0100 Subject: [PATCH] Override fixes and general updates (#129) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Kalen Michael --- tests/test_model.py | 22 +++++--------- ultralytics/hub/utils.py | 28 ++++++++++-------- ultralytics/nn/tasks.py | 8 ++--- ultralytics/yolo/configs/default.yaml | 4 +-- ultralytics/yolo/engine/exporter.py | 4 +-- ultralytics/yolo/engine/model.py | 3 +- ultralytics/yolo/engine/trainer.py | 2 ++ ultralytics/yolo/utils/__init__.py | 39 ++++++++++++++++--------- ultralytics/yolo/utils/callbacks/hub.py | 2 +- ultralytics/yolo/utils/torch_utils.py | 2 +- 10 files changed, 64 insertions(+), 50 deletions(-) diff --git a/tests/test_model.py b/tests/test_model.py index 3277bbd..7089993 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -6,16 +6,11 @@ from ultralytics import YOLO def test_model_init(): model = YOLO("yolov8n.yaml") model.info() - try: - YOLO() - except Exception: - print("Successfully caught constructor assert!") - raise Exception("constructor error didn't occur") def test_model_forward(): model = YOLO("yolov8n.yaml") - img = torch.rand(512 * 512 * 3).view(1, 3, 512, 512) + img = torch.rand(1, 3, 320, 320) model.forward(img) model(img) @@ -23,24 +18,24 @@ def test_model_forward(): def test_model_info(): model = YOLO("yolov8n.yaml") model.info() - model = model.load("best.pt") + model = YOLO("yolov8n.pt") model.info(verbose=True) def test_model_fuse(): model = YOLO("yolov8n.yaml") model.fuse() - model.load("best.pt") + model = YOLO("yolov8n.pt") model.fuse() def test_visualize_preds(): - model = YOLO("best.pt") + model = YOLO("yolov8n.pt") model.predict(source="ultralytics/assets") def test_val(): - model = YOLO("best.pt") + model = YOLO("yolov8n.pt") model.val(data="coco128.yaml", imgsz=32) @@ -54,11 +49,11 @@ def test_model_resume(): def test_model_train_pretrained(): - model = YOLO("best.pt") + model = YOLO("yolov8n.pt") model.train(data="coco128.yaml", epochs=1, imgsz=32) - model = model.new("yolov8n.yaml") + model = YOLO("yolov8n.yaml") model.train(data="coco128.yaml", epochs=1, imgsz=32) - img = torch.rand(512 * 512 * 3).view(1, 3, 512, 512) + img = torch.rand(1, 3, 320, 320) model(img) @@ -78,7 +73,6 @@ def test_exports(): 10 TensorFlow.js tfjs _web_model False False 11 PaddlePaddle paddle _paddle_model True True """ - from ultralytics import YOLO from ultralytics.yolo.engine.exporter import export_formats print(export_formats()) diff --git a/ultralytics/hub/utils.py b/ultralytics/hub/utils.py index bc6dd69..348d0a4 100644 --- a/ultralytics/hub/utils.py +++ b/ultralytics/hub/utils.py @@ -1,15 +1,15 @@ import shutil import threading import time -import uuid import requests from ultralytics.hub.config import HUB_API_ROOT -from ultralytics.yolo.utils import LOGGER, RANK, SETTINGS, colorstr, emojis +from ultralytics.yolo.utils import DEFAULT_CONFIG, LOGGER, RANK, SETTINGS, colorstr, emojis, yaml_load PREFIX = colorstr('Ultralytics: ') HELP_MSG = 'If this issue persists please visit https://github.com/ultralytics/hub/issues for assistance.' +DEFAULT_CONFIG_DICT = yaml_load(DEFAULT_CONFIG) def check_dataset_disk_space(url='https://github.com/ultralytics/yolov5/releases/download/v1.0/coco128.zip', sf=2.0): @@ -26,7 +26,7 @@ def check_dataset_disk_space(url='https://github.com/ultralytics/yolov5/releases def request_with_credentials(url: str) -> any: - """ Make a ajax request with cookies attached """ + """ Make an ajax request with cookies attached """ from google.colab import output # noqa from IPython import display # noqa display.display( @@ -92,17 +92,18 @@ def smart_request(*args, retry=3, timeout=30, thread=True, code=-1, method="post retry_codes = (408, 500) # retry only these codes methods = {'post': requests.post, 'get': requests.get} # request methods - def fcn(*args, **kwargs): - t0 = time.time() + def func(*func_args, **func_kwargs): + r = None # response + t0 = time.time() # initial time for timer for i in range(retry + 1): if (time.time() - t0) > timeout: break - r = methods[method](*args, **kwargs) # i.e. post(url, data, json, files) + r = methods[method](*func_args, **func_kwargs) # i.e. post(url, data, json, files) if r.status_code == 200: break try: m = r.json().get('message', 'No JSON message.') - except Exception: + except AttributeError: m = 'Unable to read JSON.' if i == 0: if r.status_code in retry_codes: @@ -118,22 +119,25 @@ def smart_request(*args, retry=3, timeout=30, thread=True, code=-1, method="post return r if thread: - threading.Thread(target=fcn, args=args, kwargs=kwargs, daemon=True).start() + threading.Thread(target=func, args=args, kwargs=kwargs, daemon=True).start() else: - return fcn(*args, **kwargs) + return func(*args, **kwargs) -def sync_analytics(cfg, enabled=False): +def sync_analytics(cfg, all_keys=False, enabled=True): """ Sync analytics data if enabled in the global settings Args: cfg (DictConfig): Configuration for the task and mode. + all_keys (bool): Sync all items, not just non-default values. enabled (bool): For debugging. """ if SETTINGS['sync'] and RANK in {-1, 0} and enabled: cfg = dict(cfg) # convert type from DictConfig to dict - cfg['uuid'] = uuid.getnode() # add the device UUID to the configuration data + if not all_keys: + cfg = {k: v for k, v in cfg.items() if v != DEFAULT_CONFIG_DICT[k]} # retain only non-default values + cfg['uuid'] = SETTINGS['uuid'] # add the device UUID to the configuration data # Send a request to the HUB API to sync the analytics data - smart_request(f'{HUB_API_ROOT}/analytics', data=cfg, headers=None, code=3, retry=0) + smart_request(f'{HUB_API_ROOT}/v1/usage/anonymous', data=cfg, headers=None, code=3, retry=0) diff --git a/ultralytics/nn/tasks.py b/ultralytics/nn/tasks.py index e58aa93..7edbb53 100644 --- a/ultralytics/nn/tasks.py +++ b/ultralytics/nn/tasks.py @@ -12,8 +12,8 @@ from ultralytics.nn.modules import (C1, C2, C3, C3TR, SPP, SPPF, Bottleneck, Bot GhostBottleneck, GhostConv, Segment) from ultralytics.yolo.utils import LOGGER, colorstr, yaml_load from ultralytics.yolo.utils.checks import check_yaml -from ultralytics.yolo.utils.torch_utils import (fuse_conv_and_bn, initialize_weights, intersect_state_dicts, - make_divisible, model_info, scale_img, time_sync) +from ultralytics.yolo.utils.torch_utils import (fuse_conv_and_bn, initialize_weights, intersect_dicts, make_divisible, + model_info, scale_img, time_sync) class BaseModel(nn.Module): @@ -150,7 +150,7 @@ class DetectionModel(BaseModel): def load(self, weights, verbose=True): csd = weights['model'].float().state_dict() # checkpoint state_dict as FP32 - csd = intersect_state_dicts(csd, self.state_dict()) # intersect + csd = intersect_dicts(csd, self.state_dict()) # intersect self.load_state_dict(csd, strict=False) # load if verbose: LOGGER.info(f'Transferred {len(csd)}/{len(self.model.state_dict())} items from pretrained weights') @@ -191,7 +191,7 @@ class ClassificationModel(BaseModel): def load(self, weights): model = weights["model"] if isinstance(weights, dict) else weights # torchvision models are not dicts csd = model.float().state_dict() - csd = intersect_state_dicts(csd, self.state_dict()) # intersect + csd = intersect_dicts(csd, self.state_dict()) # intersect self.load_state_dict(csd, strict=False) # load @staticmethod diff --git a/ultralytics/yolo/configs/default.yaml b/ultralytics/yolo/configs/default.yaml index 47d1659..56dd116 100644 --- a/ultralytics/yolo/configs/default.yaml +++ b/ultralytics/yolo/configs/default.yaml @@ -1,11 +1,11 @@ # YOLO 🚀 by Ultralytics, GPL-3.0 license # Default training settings and hyperparameters for medium-augmentation COCO training -task: "classify" # choices=['detect', 'segment', 'classify', 'init'] # init is a special case. Specify task to run. +task: "detect" # choices=['detect', 'segment', 'classify', 'init'] # init is a special case. Specify task to run. mode: "train" # choices=['train', 'val', 'predict'] # mode to run task in. # Train settings ------------------------------------------------------------------------------------------------------- -model: null # i.e. yolov5s.pt, yolo.yaml. Path to model file +model: null # i.e. yolov8n.pt, yolov8n.yaml. Path to model file data: null # i.e. coco128.yaml. Path to data file epochs: 100 # number of epochs to train for patience: 50 # TODO: epochs to wait for no observable improvement for early stopping of training diff --git a/ultralytics/yolo/engine/exporter.py b/ultralytics/yolo/engine/exporter.py index e518841..f422ce3 100644 --- a/ultralytics/yolo/engine/exporter.py +++ b/ultralytics/yolo/engine/exporter.py @@ -137,8 +137,6 @@ class Exporter: """ if overrides is None: overrides = {} - if 'batch_size' not in overrides: - overrides['batch_size'] = 1 # set default export batch size self.args = get_config(config, overrides) project = self.args.project or f"runs/{self.args.task}" name = self.args.name or "exp" # hardcode mode as export doesn't require it @@ -166,6 +164,8 @@ class Exporter: assert not self.args.dynamic, '--half not compatible with --dynamic, i.e. use either --half or --dynamic' # Checks + if self.args.batch_size == 16: + self.args.batch_size = 1 # TODO: resolve batch_size 16 default in config.yaml self.imgsz = check_imgsz(self.args.imgsz, stride=model.stride, min_dim=2) # check image size if self.args.optimize: assert self.device.type == 'cpu', '--optimize not compatible with cuda devices, i.e. use --device cpu' diff --git a/ultralytics/yolo/engine/model.py b/ultralytics/yolo/engine/model.py index 20b9514..0712f42 100644 --- a/ultralytics/yolo/engine/model.py +++ b/ultralytics/yolo/engine/model.py @@ -1,6 +1,7 @@ -import torch from pathlib import Path +import torch + from ultralytics import yolo # noqa from ultralytics.nn.tasks import ClassificationModel, DetectionModel, SegmentationModel, attempt_load_weights from ultralytics.yolo.configs import get_config diff --git a/ultralytics/yolo/engine/trainer.py b/ultralytics/yolo/engine/trainer.py index ba144a9..e3463ab 100644 --- a/ultralytics/yolo/engine/trainer.py +++ b/ultralytics/yolo/engine/trainer.py @@ -107,6 +107,8 @@ class BaseTrainer: self.device = utils.torch_utils.select_device(self.args.device, self.batch_size) self.amp = self.device.type != 'cpu' self.scaler = amp.GradScaler(enabled=self.amp) + if self.device.type == 'cpu': + self.args.workers = 0 # faster CPU training as time dominated by inference, not dataloading # Model and Dataloaders. self.model = self.args.model diff --git a/ultralytics/yolo/utils/__init__.py b/ultralytics/yolo/utils/__init__.py index 856358c..c788987 100644 --- a/ultralytics/yolo/utils/__init__.py +++ b/ultralytics/yolo/utils/__init__.py @@ -6,6 +6,7 @@ import platform import sys import tempfile import threading +import uuid from pathlib import Path import cv2 @@ -160,7 +161,7 @@ def get_user_config_dir(sub_dir='Ultralytics'): raise ValueError(f'Unsupported operating system: {os_name}') # GCP and AWS lambda fix, only /tmp is writeable - if not is_dir_writeable(path.parent): + if not is_dir_writeable(str(path.parent)): path = Path('/tmp') / sub_dir # Create the subdirectory if it does not exist @@ -172,9 +173,9 @@ def get_user_config_dir(sub_dir='Ultralytics'): USER_CONFIG_DIR = get_user_config_dir() # Ultralytics settings dir -def emojis(str=''): +def emojis(string=''): # Return platform-dependent emoji-safe version of string - return str.encode().decode('ascii', 'ignore') if platform.system() == 'Windows' else str + return string.encode().decode('ascii', 'ignore') if platform.system() == 'Windows' else string def colorstr(*input): @@ -282,27 +283,39 @@ def yaml_load(file='data.yaml'): """ with open(file, errors='ignore') as f: # Add YAML filename to dict and return - return {**yaml.safe_load(f), 'yaml_file': file} + return {**yaml.safe_load(f), 'yaml_file': str(file)} def get_settings(file=USER_CONFIG_DIR / 'settings.yaml'): """ - Function that loads a global settings YAML, or creates it and populates it with default values if it does not exist. + Loads a global settings YAML file or creates one with default values if it does not exist. - If the datasets or weights directories are set to None, the current working directory will be used. - The 'sync' setting determines whether analytics will be synced to help with YOLO development. + Args: + file (Path): Path to the settings YAML file. Defaults to 'settings.yaml' in the USER_CONFIG_DIR. + + Returns: + dict: Dictionary of settings key-value pairs. """ from ultralytics.yolo.utils.torch_utils import torch_distributed_zero_first + defaults = { + 'datasets_dir': None, # default datasets directory. If None, current working directory is used. + 'weights_dir': None, # default weights directory. If None, current working directory is used. + 'runs_dir': None, # default runs directory. If None, current working directory is used. + 'sync': True, # sync analytics to help with YOLO development + 'uuid': uuid.getnode(), # device UUID to align analytics + 'yaml_file': str(file)} # setting YAML file path + with torch_distributed_zero_first(RANK): if not file.exists(): - settings = { - 'datasets_dir': None, # default datasets directory. If None, current working directory is used. - 'weights_dir': None, # default weights directory. If None, current working directory is used. - 'sync': True} # sync analytics to help with YOLO development - yaml_save(file, settings) + yaml_save(file, defaults) + + settings = yaml_load(file) + if settings.keys() != defaults.keys(): + settings = {**defaults, **settings} # merge **defaults with **settings (prefer **settings) + yaml_save(file, settings) # save updated defaults - return yaml_load(file) + return settings # Run below code on utils init ----------------------------------------------------------------------------------------- diff --git a/ultralytics/yolo/utils/callbacks/hub.py b/ultralytics/yolo/utils/callbacks/hub.py index a26074e..f4cf68f 100644 --- a/ultralytics/yolo/utils/callbacks/hub.py +++ b/ultralytics/yolo/utils/callbacks/hub.py @@ -48,7 +48,7 @@ def on_train_end(trainer): # Upload final model and metrics with exponential standoff LOGGER.info(f"{PREFIX}Training completed successfully ✅\n" f"{PREFIX}Uploading final {session.model_id}") - session.upload_model(trainer.epoch, trainer.best, map=trainer.metrics['metrics/mAP50(B)'], final=True) + session.upload_model(trainer.epoch, trainer.best, map=trainer.metrics['metrics/mAP50-95(B)'], final=True) session.alive = False # stop heartbeats LOGGER.info(f"{PREFIX}View model at https://hub.ultralytics.com/models/{session.model_id} 🚀") diff --git a/ultralytics/yolo/utils/torch_utils.py b/ultralytics/yolo/utils/torch_utils.py index dd21482..b7a9863 100644 --- a/ultralytics/yolo/utils/torch_utils.py +++ b/ultralytics/yolo/utils/torch_utils.py @@ -201,7 +201,7 @@ def copy_attr(a, b, include=(), exclude=()): setattr(a, k, v) -def intersect_state_dicts(da, db, exclude=()): +def intersect_dicts(da, db, exclude=()): # Dictionary intersection of matching keys and shapes, omitting 'exclude' keys, using da values return {k: v for k, v in da.items() if k in db and all(x not in k for x in exclude) and v.shape == db[k].shape}