xdobro23/YOLOv8-16bit
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ultralytics 8.0.34 security updates and fixes (#924)

Browse Source 
Co-authored-by: Laughing <61612323+Laughing-q@users.noreply.github.com>
Co-authored-by: Fabian Greavu <fabiangreavu@gmail.com>
Co-authored-by: Yonghye Kwon <developer.0hye@gmail.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Glenn Jocher
2023-02-11 01:26:44 +04:00
committed by GitHub
parent 254adfa652
commit d32b339373
31 changed files with 134 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
ultralytics/yolo/cfg/__init__.py
View File

@ -8,9 +8,9 @@ from pathlib import Path
from types import SimpleNamespace
from typing import Dict, List, Union
from ultralytics import __version__
from ultralytics.yolo.utils import (DEFAULT_CFG, DEFAULT_CFG_DICT, DEFAULT_CFG_PATH, LOGGER, PREFIX, ROOT,
USER_CONFIG_DIR, IterableSimpleNamespace, colorstr, emojis, yaml_load, yaml_print)
USER_CONFIG_DIR, IterableSimpleNamespace, __version__, colorstr, emojis, yaml_load,
yaml_print)
from ultralytics.yolo.utils.checks import check_yolo
CLI_HELP_MSG = \
@ -25,13 +25,13 @@ CLI_HELP_MSG = \
See all ARGS at https://docs.ultralytics.com/cfg or with 'yolo cfg'
1. Train a detection model for 10 epochs with an initial learning_rate of 0.01
yolo detect train data=coco128.yaml model=yolov8n.pt epochs=10 lr0=0.01
yolo train data=coco128.yaml model=yolov8n.pt epochs=10 lr0=0.01
2. Predict a YouTube video using a pretrained segmentation model at image size 320:
yolo segment predict model=yolov8n-seg.pt source='https://youtu.be/Zgi9g1ksQHc' imgsz=320
yolo predict model=yolov8n-seg.pt source='https://youtu.be/Zgi9g1ksQHc' imgsz=320
3. Val a pretrained detection model at batch-size 1 and image size 640:
yolo detect val model=yolov8n.pt data=coco128.yaml batch=1 imgsz=640
yolo val model=yolov8n.pt data=coco128.yaml batch=1 imgsz=640
4. Export a YOLOv8n classification model to ONNX format at image size 224 by 128 (no TASK required)
yolo export model=yolov8n-cls.pt format=onnx imgsz=224,128
@ -56,7 +56,7 @@ CFG_FRACTION_KEYS = {
'mixup', 'copy_paste', 'conf', 'iou'}
CFG_INT_KEYS = {
'epochs', 'patience', 'batch', 'workers', 'seed', 'close_mosaic', 'mask_ratio', 'max_det', 'vid_stride',
'line_thickness', 'workspace', 'nbs'}
'line_thickness', 'workspace', 'nbs', 'save_period'}
CFG_BOOL_KEYS = {
'save', 'exist_ok', 'pretrained', 'verbose', 'deterministic', 'single_cls', 'image_weights', 'rect', 'cos_lr',
'overlap_mask', 'val', 'save_json', 'save_hybrid', 'half', 'dnn', 'plots', 'show', 'save_txt', 'save_conf',
@ -131,7 +131,7 @@ def get_cfg(cfg: Union[str, Path, Dict, SimpleNamespace] = DEFAULT_CFG, override
return IterableSimpleNamespace(**cfg)
def check_cfg_mismatch(base: Dict, custom: Dict):
def check_cfg_mismatch(base: Dict, custom: Dict, e=None):
"""
This function checks for any mismatched keys between a custom configuration list and a base configuration list.
If any mismatched keys are found, the function prints out similar keys from the base list and exits the program.
@ -143,12 +143,12 @@ def check_cfg_mismatch(base: Dict, custom: Dict):
base, custom = (set(x.keys()) for x in (base, custom))
mismatched = [x for x in custom if x not in base]
if mismatched:
string = ''
for x in mismatched:
matches = get_close_matches(x, base, 3, 0.6)
match_str = f"Similar arguments are {matches}." if matches else 'There are no similar arguments.'
LOGGER.warning(f"'{colorstr('red', 'bold', x)}' is not a valid YOLO argument. {match_str}")
LOGGER.warning(CLI_HELP_MSG)
sys.exit()
matches = get_close_matches(x, base)
match_str = f"Similar arguments are {matches}." if matches else ''
string += f"'{colorstr('red', 'bold', x)}' is not a valid YOLO argument. {match_str}\n"
raise SyntaxError(string + CLI_HELP_MSG) from e
def merge_equals_args(args: List[str]) -> List[str]:
@ -178,10 +178,6 @@ def merge_equals_args(args: List[str]) -> List[str]:
return new_args
def argument_error(arg):
return SyntaxError(f"'{arg}' is not a valid YOLO argument.\n{CLI_HELP_MSG}")
def entrypoint(debug=''):
"""
This function is the ultralytics package entrypoint, it's responsible for parsing the command line arguments passed
@ -212,6 +208,7 @@ def entrypoint(debug=''):
'settings': lambda: yaml_print(USER_CONFIG_DIR / 'settings.yaml'),
'cfg': lambda: yaml_print(DEFAULT_CFG_PATH),
'copy-cfg': copy_default_cfg}
FULL_ARGS_DICT = {**DEFAULT_CFG_DICT, **{k: None for k in tasks}, **{k: None for k in modes}, **special}
special = {**special, **{f'-{k}': v for k, v in special.items()}, **{f'--{k}': v for k, v in special.items()}}
overrides = {} # basic overrides, i.e. imgsz=320
@ -236,7 +233,7 @@ def entrypoint(debug=''):
v = eval(v)
overrides[k] = v
except (NameError, SyntaxError, ValueError, AssertionError) as e:
raise argument_error(a) from e
check_cfg_mismatch(FULL_ARGS_DICT, {a: ""}, e)
elif a in tasks:
overrides['task'] = a
@ -251,7 +248,7 @@ def entrypoint(debug=''):
raise SyntaxError(f"'{colorstr('red', 'bold', a)}' is a valid YOLO argument but is missing an '=' sign "
f"to set its value, i.e. try '{a}={DEFAULT_CFG_DICT[a]}'\n{CLI_HELP_MSG}")
else:
raise argument_error(a)
check_cfg_mismatch(FULL_ARGS_DICT, {a: ""})
# Defaults
task2model = dict(detect='yolov8n.pt', segment='yolov8n-seg.pt', classify='yolov8n-cls.pt')

1
ultralytics/yolo/cfg/default.yaml
View File

@ -12,6 +12,7 @@ patience: 50 # epochs to wait for no observable improvement for early stopping
batch: 16 # number of images per batch (-1 for AutoBatch)
imgsz: 640 # size of input images as integer or w,h
save: True # save train checkpoints and predict results
save_period: -1 # Save checkpoint every x epochs (disabled if < 1)
cache: False # True/ram, disk or False. Use cache for data loading
device: # device to run on, i.e. cuda device=0 or device=0,1,2,3 or device=cpu
workers: 8 # number of worker threads for data loading (per RANK if DDP)

2
ultralytics/yolo/data/dataloaders/stream_loaders.py
View File

@ -291,7 +291,7 @@ class LoadPilAndNumpy:
self.mode = 'image'
# generate fake paths
self.paths = [f"image{i}.jpg" for i in range(len(self.im0))]
self.bs = 1
self.bs = len(self.im0)
@staticmethod
def _single_check(im):

1
ultralytics/yolo/data/dataloaders/v5loader.py
View File

@ -34,6 +34,7 @@ from ultralytics.yolo.utils.checks import check_requirements, check_yaml
from ultralytics.yolo.utils.downloads import unzip_file
from ultralytics.yolo.utils.ops import clean_str, segments2boxes, xyn2xy, xywh2xyxy, xywhn2xyxy, xyxy2xywhn
from ultralytics.yolo.utils.torch_utils import torch_distributed_zero_first
from .v5augmentations import (Albumentations, augment_hsv, classify_albumentations, classify_transforms, copy_paste,
letterbox, mixup, random_perspective)

2
ultralytics/yolo/data/dataset.py
View File

@ -85,7 +85,6 @@ class YOLODataset(BaseDataset):
x["results"] = nf, nm, ne, nc, len(self.im_files)
x["msgs"] = msgs # warnings
x["version"] = self.cache_version # cache version
self.im_files = [lb["im_file"] for lb in x["labels"]] # update im_files
if is_dir_writeable(path.parent):
np.save(str(path), x) # save cache for next time
path.with_suffix(".cache.npy").rename(path) # remove .npy suffix
@ -117,6 +116,7 @@ class YOLODataset(BaseDataset):
# Read cache
[cache.pop(k) for k in ("hash", "version", "msgs")] # remove items
labels = cache["labels"]
self.im_files = [lb["im_file"] for lb in labels] # update im_files
# Check if the dataset is all boxes or all segments
len_cls = sum(len(lb["cls"]) for lb in labels)

7
ultralytics/yolo/engine/exporter.py
View File

@ -64,14 +64,13 @@ import numpy as np
import pandas as pd
import torch
import ultralytics
from ultralytics.nn.autobackend import check_class_names
from ultralytics.nn.modules import Detect, Segment
from ultralytics.nn.tasks import DetectionModel, SegmentationModel
from ultralytics.yolo.cfg import get_cfg
from ultralytics.yolo.data.dataloaders.stream_loaders import LoadImages
from ultralytics.yolo.data.utils import check_det_dataset, IMAGENET_MEAN, IMAGENET_STD
from ultralytics.yolo.utils import DEFAULT_CFG, LOGGER, callbacks, colorstr, get_default_args, yaml_save
from ultralytics.yolo.data.utils import IMAGENET_MEAN, IMAGENET_STD, check_det_dataset
from ultralytics.yolo.utils import DEFAULT_CFG, LOGGER, __version__, callbacks, colorstr, get_default_args, yaml_save
from ultralytics.yolo.utils.checks import check_imgsz, check_requirements, check_version, check_yaml
from ultralytics.yolo.utils.files import file_size
from ultralytics.yolo.utils.ops import Profile
@ -209,7 +208,7 @@ class Exporter:
'description': f"Ultralytics {self.pretty_name} model trained on {self.model.args['data']}",
'author': 'Ultralytics',
'license': 'GPL-3.0 https://ultralytics.com/license',
'version': ultralytics.__version__,
'version': __version__,
'stride': int(max(model.stride)),
'names': model.names} # model metadata

10
ultralytics/yolo/engine/model.py
View File

@ -1,16 +1,17 @@
# Ultralytics YOLO 🚀, GPL-3.0 license
import sys
from pathlib import Path
from typing import List
import sys
from ultralytics import yolo # noqa
from ultralytics.nn.tasks import (ClassificationModel, DetectionModel, SegmentationModel, attempt_load_one_weight,
guess_model_task)
from ultralytics.yolo.cfg import get_cfg
from ultralytics.yolo.engine.exporter import Exporter
from ultralytics.yolo.utils import DEFAULT_CFG, LOGGER, RANK, callbacks, yaml_load
from ultralytics.yolo.utils.checks import check_yaml, check_imgsz
from ultralytics.yolo.utils.checks import check_imgsz, check_yaml
from ultralytics.yolo.utils.downloads import GITHUB_ASSET_STEMS
from ultralytics.yolo.utils.torch_utils import smart_inference_mode
# Map head to model, trainer, validator, and predictor classes
@ -58,10 +59,13 @@ class YOLO:
# Load or create new YOLO model
load_methods = {'.pt': self._load, '.yaml': self._new}
suffix = Path(model).suffix
if not suffix and Path(model).stem in GITHUB_ASSET_STEMS:
model, suffix = Path(model).with_suffix('.pt'), '.pt' # add suffix, i.e. yolov8n -> yolov8n.pt
if suffix in load_methods:
{'.pt': self._load, '.yaml': self._new}[suffix](model)
else:
raise NotImplementedError(f"'{suffix}' model loading not implemented")
raise NotImplementedError(f"'{suffix}' models not supported. Try a *.pt and *.yaml model, "
"i.e. model='yolov8n.pt' or model='yolov8n.yaml'")
def __call__(self, source=None, stream=False, **kwargs):
return self.predict(source, stream, **kwargs)

9
ultralytics/yolo/engine/trainer.py
View File

@ -20,12 +20,11 @@ from torch.nn.parallel import DistributedDataParallel as DDP
from torch.optim import lr_scheduler
from tqdm import tqdm
from ultralytics import __version__
from ultralytics.nn.tasks import attempt_load_one_weight, attempt_load_weights
from ultralytics.yolo.cfg import get_cfg
from ultralytics.yolo.data.utils import check_cls_dataset, check_det_dataset
from ultralytics.yolo.utils import (DEFAULT_CFG, LOGGER, RANK, SETTINGS, TQDM_BAR_FORMAT, callbacks, colorstr, emojis,
yaml_save)
from ultralytics.yolo.utils import (DEFAULT_CFG, LOGGER, RANK, SETTINGS, TQDM_BAR_FORMAT, __version__, callbacks,
colorstr, emojis, yaml_save)
from ultralytics.yolo.utils.autobatch import check_train_batch_size
from ultralytics.yolo.utils.checks import check_file, check_imgsz, print_args
from ultralytics.yolo.utils.dist import ddp_cleanup, generate_ddp_command
@ -51,6 +50,7 @@ class BaseTrainer:
wdir (Path): Directory to save weights.
last (Path): Path to last checkpoint.
best (Path): Path to best checkpoint.
save_period (int): Save checkpoint every x epochs (disabled if < 1).
batch_size (int): Batch size for training.
epochs (int): Number of epochs to train for.
start_epoch (int): Starting epoch for training.
@ -101,6 +101,7 @@ class BaseTrainer:
self.args.save_dir = str(self.save_dir)
yaml_save(self.save_dir / 'args.yaml', vars(self.args)) # save run args
self.last, self.best = self.wdir / 'last.pt', self.wdir / 'best.pt' # checkpoint paths
self.save_period = self.args.save_period
self.batch_size = self.args.batch
self.epochs = self.args.epochs
@ -392,6 +393,8 @@ class BaseTrainer:
torch.save(ckpt, self.last)
if self.best_fitness == self.fitness:
torch.save(ckpt, self.best)
if (self.epoch > 0) and (self.save_period > 0) and (self.epoch % self.save_period == 0):
torch.save(ckpt, self.wdir / f'epoch{self.epoch}.pt')
del ckpt
def get_dataset(self, data):

16
ultralytics/yolo/utils/__init__.py
View File

@ -22,6 +22,8 @@ import requests
import torch
import yaml
from ultralytics import __version__
# Constants
FILE = Path(__file__).resolve()
ROOT = FILE.parents[2] # YOLO
@ -520,30 +522,27 @@ def set_sentry():
if 'exc_info' in hint:
exc_type, exc_value, tb = hint['exc_info']
if exc_type in (KeyboardInterrupt, FileNotFoundError) \
or 'out of memory' in str(exc_value) \
or not sys.argv[0].endswith('yolo'):
or 'out of memory' in str(exc_value):
return None # do not send event
env = 'Colab' if is_colab() else 'Kaggle' if is_kaggle() else 'Jupyter' if is_jupyter() else \
'Docker' if is_docker() else platform.system()
event['tags'] = {
"sys_argv": sys.argv[0],
"sys_argv_name": Path(sys.argv[0]).name,
"install": 'git' if is_git_dir() else 'pip' if is_pip_package() else 'other',
"os": env}
"os": ENVIRONMENT}
return event
if SETTINGS['sync'] and \
RANK in {-1, 0} and \
sys.argv[0].endswith('yolo') and \
Path(sys.argv[0]).name == 'yolo' and \
not is_pytest_running() and \
not is_github_actions_ci() and \
((is_pip_package() and not is_git_dir()) or
(get_git_origin_url() == "https://github.com/ultralytics/ultralytics.git" and get_git_branch() == "main")):
import hashlib
import sentry_sdk # noqa
from ultralytics import __version__
sentry_sdk.init(
dsn="https://f805855f03bb4363bc1e16cb7d87b654@o4504521589325824.ingest.sentry.io/4504521592406016",
@ -572,6 +571,7 @@ def get_settings(file=USER_CONFIG_DIR / 'settings.yaml', version='0.0.2'):
dict: Dictionary of settings key-value pairs.
"""
import hashlib
from ultralytics.yolo.utils.checks import check_version
from ultralytics.yolo.utils.torch_utils import torch_distributed_zero_first
@ -628,4 +628,6 @@ if platform.system() == 'Windows':
PREFIX = colorstr("Ultralytics: ")
SETTINGS = get_settings()
DATASETS_DIR = Path(SETTINGS['datasets_dir']) # global datasets directory
ENVIRONMENT = 'Colab' if is_colab() else 'Kaggle' if is_kaggle() else 'Jupyter' if is_jupyter() else \
'Docker' if is_docker() else platform.system()
set_sentry()

2
ultralytics/yolo/utils/checks.py
View File

@ -155,7 +155,7 @@ def check_online() -> bool:
bool: True if connection is successful, False otherwise.
"""
import socket
with contextlib.suppress(subprocess.CalledProcessError):
with contextlib.suppress(Exception):
host = socket.gethostbyname("www.github.com")
socket.create_connection((host, 80), timeout=2)
return True

11
ultralytics/yolo/utils/downloads.py
View File

@ -6,7 +6,7 @@ from itertools import repeat
from multiprocessing.pool import ThreadPool
from pathlib import Path
from urllib import parse, request
from zipfile import ZipFile, is_zipfile, BadZipFile
from zipfile import BadZipFile, ZipFile, is_zipfile
import requests
import torch
@ -14,6 +14,11 @@ from tqdm import tqdm
from ultralytics.yolo.utils import LOGGER
GITHUB_ASSET_NAMES = [f'yolov8{size}{suffix}.pt' for size in 'nsmlx' for suffix in ('', '6', '-cls', '-seg')] + \
[f'yolov5{size}u.pt' for size in 'nsmlx'] + \
[f'yolov3{size}u.pt' for size in ('', '-spp', '-tiny')]
GITHUB_ASSET_STEMS = [Path(k).stem for k in GITHUB_ASSET_NAMES]
def is_url(url, check=True):
# Check if string is URL and check if URL exists
@ -158,9 +163,7 @@ def attempt_download_asset(file, repo='ultralytics/assets', release='v0.0.0'):
return file
# GitHub assets
assets = [f'yolov8{size}{suffix}.pt' for size in 'nsmlx' for suffix in ('', '6', '-cls', '-seg')] + \
[f'yolov5{size}u.pt' for size in 'nsmlx'] + \
[f'yolov3{size}u.pt' for size in ('', '-spp', '-tiny')]
assets = GITHUB_ASSET_NAMES
try:
tag, assets = github_assets(repo, release)
except Exception:

3
ultralytics/yolo/utils/torch_utils.py
View File

@ -17,7 +17,7 @@ import torch.nn as nn
import torch.nn.functional as F
from torch.nn.parallel import DistributedDataParallel as DDP
from ultralytics.yolo.utils import DEFAULT_CFG_DICT, DEFAULT_CFG_KEYS, LOGGER
from ultralytics.yolo.utils import DEFAULT_CFG_DICT, DEFAULT_CFG_KEYS, LOGGER, __version__
from ultralytics.yolo.utils.checks import check_version
LOCAL_RANK = int(os.getenv('LOCAL_RANK', -1)) # https://pytorch.org/docs/stable/elastic/run.html
@ -61,7 +61,6 @@ def DDP_model(model):
def select_device(device='', batch=0, newline=False):
# device = None or 'cpu' or 0 or '0' or '0,1,2,3'
from ultralytics import __version__
s = f"Ultralytics YOLOv{__version__} 🚀 Python-{platform.python_version()} torch-{torch.__version__} "
device = str(device).lower()
for remove in 'cuda:', 'none', '(', ')', '[', ']', "'", ' ':

2
ultralytics/yolo/v8/classify/train.py
View File

@ -8,7 +8,7 @@ from ultralytics.yolo import v8
from ultralytics.yolo.data import build_classification_dataloader
from ultralytics.yolo.engine.trainer import BaseTrainer
from ultralytics.yolo.utils import DEFAULT_CFG
from ultralytics.yolo.utils.torch_utils import strip_optimizer, is_parallel
from ultralytics.yolo.utils.torch_utils import is_parallel, strip_optimizer
class ClassificationTrainer(BaseTrainer):