xdobro23/YOLOv8-16bit
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ultralytics 8.0.141 create new SettingsManager (#3790)

Browse Source
This commit is contained in:
Glenn Jocher
2023-07-23 16:03:34 +02:00
committed by GitHub
parent 42afe772d5
commit 20f5efd40a
215 changed files with 917 additions and 749 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
ultralytics/__init__.py
View File

@ -1,13 +1,14 @@
# Ultralytics YOLO 🚀, AGPL-3.0 license
__version__ = '8.0.140'
__version__ = '8.0.141'
from ultralytics.engine.model import YOLO
from ultralytics.hub import start
from ultralytics.models import RTDETR, SAM
from ultralytics.models.fastsam import FastSAM
from ultralytics.models.nas import NAS
from ultralytics.utils import SETTINGS as settings
from ultralytics.utils.checks import check_yolo as checks
from ultralytics.utils.downloads import download
__all__ = '__version__', 'YOLO', 'NAS', 'SAM', 'FastSAM', 'RTDETR', 'checks', 'download', 'start' # allow simpler import
__all__ = '__version__', 'YOLO', 'NAS', 'SAM', 'FastSAM', 'RTDETR', 'checks', 'download', 'start', 'settings' # allow simpler import

83
ultralytics/cfg/__init__.py
View File

@ -9,9 +9,9 @@ from pathlib import Path
from types import SimpleNamespace
from typing import Dict, List, Union
from ultralytics.utils import (DEFAULT_CFG, DEFAULT_CFG_DICT, DEFAULT_CFG_PATH, LOGGER, ROOT, USER_CONFIG_DIR,
IterableSimpleNamespace, __version__, checks, colorstr, deprecation_warn, get_settings,
yaml_load, yaml_print)
from ultralytics.utils import (DEFAULT_CFG, DEFAULT_CFG_DICT, DEFAULT_CFG_PATH, LOGGER, ROOT, SETTINGS, SETTINGS_YAML,
IterableSimpleNamespace, __version__, checks, colorstr, deprecation_warn, yaml_load,
yaml_print)
# Define valid tasks and modes
MODES = 'train', 'val', 'predict', 'export', 'track', 'benchmark'
@ -28,7 +28,6 @@ TASK2METRIC = {
'classify': 'metrics/accuracy_top1',
'pose': 'metrics/mAP50-95(P)'}
CLI_HELP_MSG = \
f"""
Arguments received: {str(['yolo'] + sys.argv[1:])}. Ultralytics 'yolo' commands use the following syntax:
@ -111,7 +110,7 @@ def get_cfg(cfg: Union[str, Path, Dict, SimpleNamespace] = DEFAULT_CFG_DICT, ove
# Merge overrides
if overrides:
overrides = cfg2dict(overrides)
check_cfg_mismatch(cfg, overrides)
check_dict_alignment(cfg, overrides)
cfg = {**cfg, **overrides} # merge cfg and overrides dicts (prefer overrides)
# Special handling for numeric project/name
@ -147,9 +146,7 @@ def get_cfg(cfg: Union[str, Path, Dict, SimpleNamespace] = DEFAULT_CFG_DICT, ove
def _handle_deprecation(custom):
"""
Hardcoded function to handle deprecated config keys
"""
"""Hardcoded function to handle deprecated config keys"""
for key in custom.copy().keys():
if key == 'hide_labels':
@ -165,7 +162,7 @@ def _handle_deprecation(custom):
return custom
def check_cfg_mismatch(base: Dict, custom: Dict, e=None):
def check_dict_alignment(base: Dict, custom: Dict, e=None):
"""
This function checks for any mismatched keys between a custom configuration list and a base configuration list.
If any mismatched keys are found, the function prints out similar keys from the base list and exits the program.
@ -175,13 +172,13 @@ def check_cfg_mismatch(base: Dict, custom: Dict, e=None):
base (dict): a dictionary of base configuration options
"""
custom = _handle_deprecation(custom)
base, custom = (set(x.keys()) for x in (base, custom))
mismatched = [x for x in custom if x not in base]
base_keys, custom_keys = (set(x.keys()) for x in (base, custom))
mismatched = [k for k in custom_keys if k not in base_keys]
if mismatched:
string = ''
for x in mismatched:
matches = get_close_matches(x, base) # key list
matches = [f'{k}={DEFAULT_CFG_DICT[k]}' if DEFAULT_CFG_DICT.get(k) is not None else k for k in matches]
matches = get_close_matches(x, base_keys) # key list
matches = [f'{k}={base[k]}' if base.get(k) is not None else k for k in matches]
match_str = f'Similar arguments are i.e. {matches}.' if matches else ''
string += f"'{colorstr('red', 'bold', x)}' is not a valid YOLO argument. {match_str}\n"
raise SyntaxError(string + CLI_HELP_MSG) from e
@ -251,12 +248,39 @@ def handle_yolo_settings(args: List[str]) -> None:
Example:
python my_script.py yolo settings reset
"""
path = USER_CONFIG_DIR / 'settings.yaml' # get SETTINGS YAML file path
if any(args) and args[0] == 'reset':
path.unlink() # delete the settings file
get_settings() # create new settings
LOGGER.info('Settings reset successfully') # inform the user that settings have been reset
yaml_print(path) # print the current settings
if any(args):
if args[0] == 'reset':
SETTINGS_YAML.unlink() # delete the settings file
SETTINGS.reset() # create new settings
LOGGER.info('Settings reset successfully') # inform the user that settings have been reset
else:
new = dict(parse_key_value_pair(a) for a in args)
check_dict_alignment(SETTINGS, new)
SETTINGS.update(new)
yaml_print(SETTINGS_YAML) # print the current settings
def parse_key_value_pair(pair):
"""Parse one 'key=value' pair and return key and value."""
re.sub(r' *= *', '=', pair) # remove spaces around equals sign
k, v = pair.split('=', 1) # split on first '=' sign
assert v, f"missing '{k}' value"
return k, smart_value(v)
def smart_value(v):
"""Convert a string to an underlying type such as int, float, bool, etc."""
if v.lower() == 'none':
return None
elif v.lower() == 'true':
return True
elif v.lower() == 'false':
return False
else:
with contextlib.suppress(Exception):
return eval(v)
return v
def entrypoint(debug=''):
@ -305,25 +329,14 @@ def entrypoint(debug=''):
a = a[:-1]
if '=' in a:
try:
re.sub(r' *= *', '=', a) # remove spaces around equals sign
k, v = a.split('=', 1) # split on first '=' sign
assert v, f"missing '{k}' value"
k, v = parse_key_value_pair(a)
if k == 'cfg': # custom.yaml passed
LOGGER.info(f'Overriding {DEFAULT_CFG_PATH} with {v}')
overrides = {k: val for k, val in yaml_load(checks.check_yaml(v)).items() if k != 'cfg'}
else:
if v.lower() == 'none':
v = None
elif v.lower() == 'true':
v = True
elif v.lower() == 'false':
v = False
else:
with contextlib.suppress(Exception):
v = eval(v)
overrides[k] = v
except (NameError, SyntaxError, ValueError, AssertionError) as e:
check_cfg_mismatch(full_args_dict, {a: ''}, e)
check_dict_alignment(full_args_dict, {a: ''}, e)
elif a in TASKS:
overrides['task'] = a
@ -338,13 +351,13 @@ def entrypoint(debug=''):
raise SyntaxError(f"'{colorstr('red', 'bold', a)}' is a valid YOLO argument but is missing an '=' sign "
f"to set its value, i.e. try '{a}={DEFAULT_CFG_DICT[a]}'\n{CLI_HELP_MSG}")
else:
check_cfg_mismatch(full_args_dict, {a: ''})
check_dict_alignment(full_args_dict, {a: ''})
# Check keys
check_cfg_mismatch(full_args_dict, overrides)
check_dict_alignment(full_args_dict, overrides)
# Mode
mode = overrides.get('mode', None)
mode = overrides.get('mode')
if mode is None:
mode = DEFAULT_CFG.mode or 'predict'
LOGGER.warning(f"WARNING ⚠️ 'mode' is missing. Valid modes are {MODES}. Using default 'mode={mode}'.")

4
ultralytics/hub/auth.py
View File

@ -3,7 +3,7 @@
import requests
from ultralytics.hub.utils import HUB_API_ROOT, PREFIX, request_with_credentials
from ultralytics.utils import LOGGER, SETTINGS, emojis, is_colab, set_settings
from ultralytics.utils import LOGGER, SETTINGS, emojis, is_colab
API_KEY_URL = 'https://hub.ultralytics.com/settings?tab=api+keys'
@ -45,7 +45,7 @@ class Auth:
# Update SETTINGS with the new API key after successful authentication
if success:
set_settings({'api_key': self.api_key})
SETTINGS.update({'api_key': self.api_key})
# Log that the new login was successful
if verbose:
LOGGER.info(f'{PREFIX}New authentication successful ✅')

137
ultralytics/utils/__init__.py
View File

@ -713,62 +713,105 @@ def set_sentry():
logging.getLogger(logger).setLevel(logging.CRITICAL)
def get_settings(file=SETTINGS_YAML, version='0.0.3'):
def update_dict_recursive(d, u):
"""
Loads a global Ultralytics settings YAML file or creates one with default values if it does not exist.
Recursively updates the dictionary `d` with the key-value pairs from the dictionary `u` without overwriting
entire sub-dictionaries. Note that function recursion is intended and not a problem, as this allows for updating
nested dictionaries at any arbitrary depth.
Args:
file (Path): Path to the Ultralytics settings YAML file. Defaults to 'settings.yaml' in the USER_CONFIG_DIR.
version (str): Settings version. If min settings version not met, new default settings will be saved.
d (dict): The dictionary to be updated.
u (dict): The dictionary to update `d` with.
Returns:
(dict): Dictionary of settings key-value pairs.
(dict): The recursively updated dictionary.
"""
import hashlib
from ultralytics.utils.checks import check_version
from ultralytics.utils.torch_utils import torch_distributed_zero_first
git_dir = get_git_dir()
root = git_dir or Path()
datasets_root = (root.parent if git_dir and is_dir_writeable(root.parent) else root).resolve()
defaults = {
'datasets_dir': str(datasets_root / 'datasets'), # default datasets directory.
'weights_dir': str(root / 'weights'), # default weights directory.
'runs_dir': str(root / 'runs'), # default runs directory.
'uuid': hashlib.sha256(str(uuid.getnode()).encode()).hexdigest(), # SHA-256 anonymized UUID hash
'sync': True, # sync analytics to help with YOLO development
'api_key': '', # Ultralytics HUB API key (https://hub.ultralytics.com/)
'settings_version': version} # Ultralytics settings version
with torch_distributed_zero_first(RANK):
if not file.exists():
yaml_save(file, defaults)
settings = yaml_load(file)
# Check that settings keys and types match defaults
correct = \
settings \
and settings.keys() == defaults.keys() \
and all(type(a) == type(b) for a, b in zip(settings.values(), defaults.values())) \
and check_version(settings['settings_version'], version)
if not correct:
LOGGER.warning('WARNING ⚠️ Ultralytics settings reset to defaults. This is normal and may be due to a '
'recent ultralytics package update, but may have overwritten previous settings. '
f"\nView and update settings with 'yolo settings' or at '{file}'")
settings = defaults # merge **defaults with **settings (prefer **settings)
yaml_save(file, settings) # save updated defaults
return settings
for k, v in u.items():
d[k] = update_dict_recursive(d.get(k, {}), v) if isinstance(v, dict) else v
return d
def set_settings(kwargs, file=SETTINGS_YAML):
class SettingsManager(dict):
"""
Function that runs on a first-time ultralytics package installation to set up global settings and create necessary
directories.
Manages Ultralytics settings stored in a YAML file.
Args:
file (str | Path): Path to the Ultralytics settings YAML file. Default is USER_CONFIG_DIR / 'settings.yaml'.
version (str): Settings version. In case of local version mismatch, new default settings will be saved.
"""
SETTINGS.update(kwargs)
yaml_save(file, SETTINGS)
def __init__(self, file=SETTINGS_YAML, version='0.0.4'):
import copy
import hashlib
from ultralytics.utils.checks import check_version
from ultralytics.utils.torch_utils import torch_distributed_zero_first
git_dir = get_git_dir()
root = git_dir or Path()
datasets_root = (root.parent if git_dir and is_dir_writeable(root.parent) else root).resolve()
self.file = Path(file)
self.version = version
self.defaults = {
'settings_version': version,
'datasets_dir': str(datasets_root / 'datasets'),
'weights_dir': str(root / 'weights'),
'runs_dir': str(root / 'runs'),
'uuid': hashlib.sha256(str(uuid.getnode()).encode()).hexdigest(),
'sync': True,
'api_key': '',
'clearml': True, # integrations
'comet': True,
'dvc': True,
'hub': True,
'mlflow': True,
'neptune': True,
'raytune': True,
'tensorboard': True,
'wandb': True}
super().__init__(copy.deepcopy(self.defaults))
with torch_distributed_zero_first(RANK):
if not self.file.exists():
self.save()
self.load()
correct_keys = self.keys() == self.defaults.keys()
correct_types = all(type(a) == type(b) for a, b in zip(self.values(), self.defaults.values()))
correct_version = check_version(self['settings_version'], self.version)
if not (correct_keys and correct_types and correct_version):
LOGGER.warning(
'WARNING ⚠️ Ultralytics settings reset to default values. This may be due to a possible problem '
'with your settings or a recent ultralytics package update. '
f"\nView settings with 'yolo settings' or at '{self.file}'"
"\nUpdate settings with 'yolo settings key=value', i.e. 'yolo settings runs_dir=path/to/dir'.")
self.reset()
def load(self):
"""Loads settings from the YAML file."""
self.update(yaml_load(self.file))
def save(self):
"""Saves the current settings to the YAML file."""
yaml_save(self.file, dict(self))
def update(self, *args, **kwargs):
"""Updates a setting value in the current settings and saves the settings."""
new = dict(*args, **kwargs)
if any(isinstance(v, dict) for v in new.values()):
update_dict_recursive(self, new)
else:
# super().update(*args, **kwargs)
super().update(new)
self.save()
def reset(self):
"""Resets the settings to default and saves them."""
self.clear()
self.update(self.defaults)
self.save()
def deprecation_warn(arg, new_arg, version=None):
@ -794,7 +837,7 @@ def url2file(url):
# Check first-install steps
PREFIX = colorstr('Ultralytics: ')
SETTINGS = get_settings()
SETTINGS = SettingsManager() # initialize settings
DATASETS_DIR = Path(SETTINGS['datasets_dir']) # global datasets directory
ENVIRONMENT = 'Colab' if is_colab() else 'Kaggle' if is_kaggle() else 'Jupyter' if is_jupyter() else \
'Docker' if is_docker() else platform.system()

3
ultralytics/utils/callbacks/clearml.py
View File

@ -5,7 +5,7 @@ import re
import matplotlib.image as mpimg
import matplotlib.pyplot as plt
from ultralytics.utils import LOGGER, TESTS_RUNNING
from ultralytics.utils import LOGGER, SETTINGS, TESTS_RUNNING
from ultralytics.utils.torch_utils import model_info_for_loggers
try:
@ -16,6 +16,7 @@ try:
assert hasattr(clearml, '__version__') # verify package is not directory
assert not TESTS_RUNNING # do not log pytest
assert SETTINGS['clearml'] is True # verify integration is enabled
except (ImportError, AssertionError):
clearml = None

3
ultralytics/utils/callbacks/comet.py
View File

@ -3,7 +3,7 @@
import os
from pathlib import Path
from ultralytics.utils import LOGGER, RANK, TESTS_RUNNING, ops
from ultralytics.utils import LOGGER, RANK, SETTINGS, TESTS_RUNNING, ops
from ultralytics.utils.torch_utils import model_info_for_loggers
try:
@ -11,6 +11,7 @@ try:
assert not TESTS_RUNNING # do not log pytest
assert hasattr(comet_ml, '__version__') # verify package is not directory
assert SETTINGS['comet'] is True # verify integration is enabled
except (ImportError, AssertionError):
comet_ml = None

3
ultralytics/utils/callbacks/dvc.py
View File

@ -3,7 +3,7 @@ import os
import pkg_resources as pkg
from ultralytics.utils import LOGGER, TESTS_RUNNING
from ultralytics.utils import LOGGER, SETTINGS, TESTS_RUNNING
from ultralytics.utils.torch_utils import model_info_for_loggers
try:
@ -12,6 +12,7 @@ try:
import dvclive
assert not TESTS_RUNNING # do not log pytest
assert SETTINGS['dvc'] is True # verify integration is enabled
ver = version('dvclive')
if pkg.parse_version(ver) < pkg.parse_version('2.11.0'):

4
ultralytics/utils/callbacks/hub.py
View File

@ -4,7 +4,7 @@ import json
from time import time
from ultralytics.hub.utils import PREFIX, events
from ultralytics.utils import LOGGER
from ultralytics.utils import LOGGER, SETTINGS
from ultralytics.utils.torch_utils import model_info_for_loggers
@ -84,4 +84,4 @@ callbacks = {
'on_train_start': on_train_start,
'on_val_start': on_val_start,
'on_predict_start': on_predict_start,
'on_export_start': on_export_start}
'on_export_start': on_export_start} if SETTINGS['hub'] is True else {} # verify enabled

3
ultralytics/utils/callbacks/mlflow.py
View File

@ -4,13 +4,14 @@ import os
import re
from pathlib import Path
from ultralytics.utils import LOGGER, TESTS_RUNNING, colorstr
from ultralytics.utils import LOGGER, SETTINGS, TESTS_RUNNING, colorstr
try:
import mlflow
assert not TESTS_RUNNING # do not log pytest
assert hasattr(mlflow, '__version__') # verify package is not directory
assert SETTINGS['mlflow'] is True # verify integration is enabled
except (ImportError, AssertionError):
mlflow = None

3
ultralytics/utils/callbacks/neptune.py
View File

@ -3,7 +3,7 @@
import matplotlib.image as mpimg
import matplotlib.pyplot as plt
from ultralytics.utils import LOGGER, TESTS_RUNNING
from ultralytics.utils import LOGGER, SETTINGS, TESTS_RUNNING
from ultralytics.utils.torch_utils import model_info_for_loggers
try:
@ -12,6 +12,7 @@ try:
assert not TESTS_RUNNING # do not log pytest
assert hasattr(neptune, '__version__')
assert SETTINGS['neptune'] is True # verify integration is enabled
except (ImportError, AssertionError):
neptune = None

4
ultralytics/utils/callbacks/raytune.py
View File

@ -1,9 +1,13 @@
# Ultralytics YOLO 🚀, AGPL-3.0 license
from ultralytics.utils import SETTINGS
try:
import ray
from ray import tune
from ray.air import session
assert SETTINGS['raytune'] is True # verify integration is enabled
except (ImportError, AssertionError):
tune = None

3
ultralytics/utils/callbacks/tensorboard.py
View File

@ -1,11 +1,12 @@
# Ultralytics YOLO 🚀, AGPL-3.0 license
from ultralytics.utils import LOGGER, TESTS_RUNNING, colorstr
from ultralytics.utils import LOGGER, SETTINGS, TESTS_RUNNING, colorstr
try:
from torch.utils.tensorboard import SummaryWriter
assert not TESTS_RUNNING # do not log pytest
assert SETTINGS['tensorboard'] is True # verify integration is enabled
# TypeError for handling 'Descriptors cannot not be created directly.' protobuf errors in Windows
except (ImportError, AssertionError, TypeError):

5
ultralytics/utils/callbacks/wb.py
View File

@ -1,5 +1,5 @@
# Ultralytics YOLO 🚀, AGPL-3.0 license
from ultralytics.utils import TESTS_RUNNING
from ultralytics.utils import SETTINGS, TESTS_RUNNING
from ultralytics.utils.torch_utils import model_info_for_loggers
try:
@ -7,6 +7,7 @@ try:
assert hasattr(wb, '__version__')
assert not TESTS_RUNNING # do not log pytest
assert SETTINGS['wandb'] is True # verify integration is enabled
except (ImportError, AssertionError):
wb = None
@ -16,7 +17,7 @@ _processed_plots = {}
def _log_plots(plots, step):
for name, params in plots.items():
timestamp = params['timestamp']
if _processed_plots.get(name, None) != timestamp:
if _processed_plots.get(name) != timestamp:
wb.run.log({name.stem: wb.Image(str(name))}, step=step)
_processed_plots[name] = timestamp

2
ultralytics/utils/files.py
View File

@ -38,7 +38,7 @@ def spaces_in_path(path):
path (str | Path): The original path.
Yields:
Path: Temporary path with spaces replaced by underscores if spaces were present, otherwise the original path.
(Path): Temporary path with spaces replaced by underscores if spaces were present, otherwise the original path.
Examples:
with spaces_in_path('/path/with spaces') as new_path: