From 254adfa652eb9abc41a8b7bdef7d6420c81a6217 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Fri, 10 Feb 2023 01:37:57 +0400 Subject: [PATCH] `ultralytics 8.0.33` security updates and fixes (#896) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Mert Can Demir --- docker/Dockerfile-arm64 | 2 +- docker/Dockerfile-cpu | 2 +- docs/app.md | 34 +++++++++++++++++-- docs/cfg.md | 25 +++++++------- ultralytics/__init__.py | 3 +- ultralytics/yolo/cfg/__init__.py | 2 ++ ultralytics/yolo/cfg/default.yaml | 1 + ultralytics/yolo/data/dataloaders/v5loader.py | 4 +-- ultralytics/yolo/data/utils.py | 2 -- ultralytics/yolo/engine/exporter.py | 2 +- ultralytics/yolo/engine/model.py | 2 +- ultralytics/yolo/engine/trainer.py | 12 ++----- ultralytics/yolo/engine/validator.py | 3 +- ultralytics/yolo/utils/__init__.py | 22 ++++++++++++ ultralytics/yolo/utils/downloads.py | 4 ++- ultralytics/yolo/utils/files.py | 11 ------ ultralytics/yolo/v8/classify/predict.py | 1 - ultralytics/yolo/v8/classify/val.py | 1 - ultralytics/yolo/v8/detect/predict.py | 1 - ultralytics/yolo/v8/detect/train.py | 1 - ultralytics/yolo/v8/detect/val.py | 7 ++-- ultralytics/yolo/v8/segment/predict.py | 2 -- ultralytics/yolo/v8/segment/train.py | 1 - ultralytics/yolo/v8/segment/val.py | 3 +- 24 files changed, 87 insertions(+), 61 deletions(-) diff --git a/docker/Dockerfile-arm64 b/docker/Dockerfile-arm64 index 44e594b..3108c5f 100644 --- a/docker/Dockerfile-arm64 +++ b/docker/Dockerfile-arm64 @@ -3,7 +3,7 @@ # Image is aarch64-compatible for Apple M1 and other ARM architectures i.e. Jetson Nano and Raspberry Pi # Start FROM Ubuntu image https://hub.docker.com/_/ubuntu -FROM arm64v8/ubuntu:20.04 +FROM arm64v8/ubuntu:rolling # Downloads to user config dir ADD https://ultralytics.com/assets/Arial.ttf https://ultralytics.com/assets/Arial.Unicode.ttf /root/.config/Ultralytics/ diff --git a/docker/Dockerfile-cpu b/docker/Dockerfile-cpu index 0b585e8..bf515e5 100644 --- a/docker/Dockerfile-cpu +++ b/docker/Dockerfile-cpu @@ -3,7 +3,7 @@ # Image is CPU-optimized for ONNX, OpenVINO and PyTorch YOLOv8 deployments # Start FROM Ubuntu image https://hub.docker.com/_/ubuntu -FROM ubuntu:20.04 +FROM ubuntu:rolling # Downloads to user config dir ADD https://ultralytics.com/assets/Arial.ttf https://ultralytics.com/assets/Arial.Unicode.ttf /root/.config/Ultralytics/ diff --git a/docs/app.md b/docs/app.md index 8532f23..e6a45aa 100644 --- a/docs/app.md +++ b/docs/app.md @@ -1,10 +1,38 @@ # Ultralytics HUB App for YOLOv8 -
- - + + +

+
+ + + + + + + + + + + + + + + + + + + + +
+
+ + CI CPU + + Open In Colab
+
Welcome to the Ultralytics HUB app for demonstrating YOLOv5 and YOLOv8 models! In this app, available on the [Apple App Store](https://apps.apple.com/xk/app/ultralytics/id1583935240) and the diff --git a/docs/cfg.md b/docs/cfg.md index cdea450..c5eca3a 100644 --- a/docs/cfg.md +++ b/docs/cfg.md @@ -154,18 +154,19 @@ process include the size and composition of the validation dataset and the speci is important to carefully tune and experiment with these settings to ensure that the model is performing well on the validation dataset and to detect and prevent overfitting. -| Key | Value | Description | -|-------------|-------|-----------------------------------------------------------------| -| save_json | False | save results to JSON file | -| save_hybrid | False | save hybrid version of labels (labels + additional predictions) | -| conf | 0.001 | object confidence threshold for detection | -| iou | 0.6 | intersection over union (IoU) threshold for NMS | -| max_det | 300 | maximum number of detections per image | -| half | True | use half precision (FP16) | -| device | null | device to run on, i.e. cuda device=0/1/2/3 or device=cpu | -| dnn | False | use OpenCV DNN for ONNX inference | -| plots | False | show plots during training | -| rect | False | support rectangular evaluation | +| Key | Value | Description | +|-------------|-------|--------------------------------------------------------------------| +| save_json | False | save results to JSON file | +| save_hybrid | False | save hybrid version of labels (labels + additional predictions) | +| conf | 0.001 | object confidence threshold for detection | +| iou | 0.6 | intersection over union (IoU) threshold for NMS | +| max_det | 300 | maximum number of detections per image | +| half | True | use half precision (FP16) | +| device | null | device to run on, i.e. cuda device=0/1/2/3 or device=cpu | +| dnn | False | use OpenCV DNN for ONNX inference | +| plots | False | show plots during training | +| rect | False | support rectangular evaluation | +| split | val | dataset split to use for validation, i.e. 'val', 'test' or 'train' | ### Export diff --git a/ultralytics/__init__.py b/ultralytics/__init__.py index 6520f00..db41f82 100644 --- a/ultralytics/__init__.py +++ b/ultralytics/__init__.py @@ -1,9 +1,8 @@ # Ultralytics YOLO 🚀, GPL-3.0 license -__version__ = "8.0.32" +__version__ = "8.0.33" from ultralytics.yolo.engine.model import YOLO -from ultralytics.yolo.utils import ops from ultralytics.yolo.utils.checks import check_yolo as checks __all__ = ["__version__", "YOLO", "hub", "checks"] # allow simpler import diff --git a/ultralytics/yolo/cfg/__init__.py b/ultralytics/yolo/cfg/__init__.py index b6a2710..34fd6f0 100644 --- a/ultralytics/yolo/cfg/__init__.py +++ b/ultralytics/yolo/cfg/__init__.py @@ -202,6 +202,7 @@ def entrypoint(debug=''): LOGGER.info(CLI_HELP_MSG) return + # Add tasks, modes, special, and special with dash keys, i.e. -help, --help tasks = 'detect', 'segment', 'classify' modes = 'train', 'val', 'predict', 'export' special = { @@ -211,6 +212,7 @@ def entrypoint(debug=''): 'settings': lambda: yaml_print(USER_CONFIG_DIR / 'settings.yaml'), 'cfg': lambda: yaml_print(DEFAULT_CFG_PATH), 'copy-cfg': copy_default_cfg} + special = {**special, **{f'-{k}': v for k, v in special.items()}, **{f'--{k}': v for k, v in special.items()}} overrides = {} # basic overrides, i.e. imgsz=320 for a in merge_equals_args(args): # merge spaces around '=' sign diff --git a/ultralytics/yolo/cfg/default.yaml b/ultralytics/yolo/cfg/default.yaml index 4f0a43e..6c0a7cd 100644 --- a/ultralytics/yolo/cfg/default.yaml +++ b/ultralytics/yolo/cfg/default.yaml @@ -38,6 +38,7 @@ dropout: 0.0 # use dropout regularization (classify train only) # Val/Test settings ---------------------------------------------------------------------------------------------------- val: True # validate/test during training +split: val # dataset split to use for validation, i.e. 'val', 'test' or 'train' save_json: False # save results to JSON file save_hybrid: False # save hybrid version of labels (labels + additional predictions) conf: # object confidence threshold for detection (default 0.25 predict, 0.001 val) diff --git a/ultralytics/yolo/data/dataloaders/v5loader.py b/ultralytics/yolo/data/dataloaders/v5loader.py index 2b75285..37c1507 100644 --- a/ultralytics/yolo/data/dataloaders/v5loader.py +++ b/ultralytics/yolo/data/dataloaders/v5loader.py @@ -27,13 +27,13 @@ from PIL import ExifTags, Image, ImageOps from torch.utils.data import DataLoader, Dataset, dataloader, distributed from tqdm import tqdm -from ultralytics.yolo.data.utils import check_det_dataset, unzip_file +from ultralytics.yolo.data.utils import check_det_dataset from ultralytics.yolo.utils import (DATASETS_DIR, LOGGER, NUM_THREADS, TQDM_BAR_FORMAT, is_colab, is_dir_writeable, is_kaggle, yaml_load) from ultralytics.yolo.utils.checks import check_requirements, check_yaml +from ultralytics.yolo.utils.downloads import unzip_file from ultralytics.yolo.utils.ops import clean_str, segments2boxes, xyn2xy, xywh2xyxy, xywhn2xyxy, xyxy2xywhn from ultralytics.yolo.utils.torch_utils import torch_distributed_zero_first - from .v5augmentations import (Albumentations, augment_hsv, classify_albumentations, classify_transforms, copy_paste, letterbox, mixup, random_perspective) diff --git a/ultralytics/yolo/data/utils.py b/ultralytics/yolo/data/utils.py index b1163f6..0d1b62a 100644 --- a/ultralytics/yolo/data/utils.py +++ b/ultralytics/yolo/data/utils.py @@ -11,13 +11,11 @@ from zipfile import is_zipfile import cv2 import numpy as np -import torch from PIL import ExifTags, Image, ImageOps from ultralytics.yolo.utils import DATASETS_DIR, LOGGER, ROOT, colorstr, emojis, yaml_load from ultralytics.yolo.utils.checks import check_file, check_font, is_ascii from ultralytics.yolo.utils.downloads import download, safe_download -from ultralytics.yolo.utils.files import unzip_file from ultralytics.yolo.utils.ops import segments2boxes HELP_URL = "See https://github.com/ultralytics/yolov5/wiki/Train-Custom-Data" diff --git a/ultralytics/yolo/engine/exporter.py b/ultralytics/yolo/engine/exporter.py index 442ae17..760312f 100644 --- a/ultralytics/yolo/engine/exporter.py +++ b/ultralytics/yolo/engine/exporter.py @@ -67,7 +67,7 @@ import torch import ultralytics from ultralytics.nn.autobackend import check_class_names from ultralytics.nn.modules import Detect, Segment -from ultralytics.nn.tasks import ClassificationModel, DetectionModel, SegmentationModel, guess_model_task +from ultralytics.nn.tasks import DetectionModel, SegmentationModel from ultralytics.yolo.cfg import get_cfg from ultralytics.yolo.data.dataloaders.stream_loaders import LoadImages from ultralytics.yolo.data.utils import check_det_dataset, IMAGENET_MEAN, IMAGENET_STD diff --git a/ultralytics/yolo/engine/model.py b/ultralytics/yolo/engine/model.py index 10db5d1..7ae9bf7 100644 --- a/ultralytics/yolo/engine/model.py +++ b/ultralytics/yolo/engine/model.py @@ -33,7 +33,7 @@ class YOLO: A python interface which emulates a model-like behaviour by wrapping trainers. """ - def __init__(self, model='yolov8n.yaml', type="v8") -> None: + def __init__(self, model='yolov8n.pt', type="v8") -> None: """ Initializes the YOLO object. diff --git a/ultralytics/yolo/engine/trainer.py b/ultralytics/yolo/engine/trainer.py index 31561d1..3724d6a 100644 --- a/ultralytics/yolo/engine/trainer.py +++ b/ultralytics/yolo/engine/trainer.py @@ -5,7 +5,6 @@ Simple training loop; Boilerplate that could apply to any arbitrary neural netwo import os import subprocess -import sys import time from collections import defaultdict from copy import deepcopy @@ -29,10 +28,10 @@ from ultralytics.yolo.utils import (DEFAULT_CFG, LOGGER, RANK, SETTINGS, TQDM_BA yaml_save) from ultralytics.yolo.utils.autobatch import check_train_batch_size from ultralytics.yolo.utils.checks import check_file, check_imgsz, print_args -from ultralytics.yolo.utils.dist import ddp_cleanup, generate_ddp_file, find_free_network_port +from ultralytics.yolo.utils.dist import ddp_cleanup, generate_ddp_command from ultralytics.yolo.utils.files import get_latest_run, increment_path from ultralytics.yolo.utils.torch_utils import (EarlyStopping, ModelEMA, de_parallel, init_seeds, one_cycle, - select_device, strip_optimizer, TORCH_1_9) + select_device, strip_optimizer) class BaseTrainer: @@ -175,12 +174,7 @@ class BaseTrainer: # Run subprocess if DDP training, else train normally if world_size > 1 and "LOCAL_RANK" not in os.environ: - # cmd, file = generate_ddp_command(world_size, self) # security vulnerability in Snyk scans - file = generate_ddp_file(self) if sys.argv[0].endswith('yolo') else os.path.abspath(sys.argv[0]) - torch_distributed_cmd = "torch.distributed.run" if TORCH_1_9 else "torch.distributed.launch" - cmd = [ - sys.executable, "-m", torch_distributed_cmd, "--nproc_per_node", f"{world_size}", "--master_port", - f"{find_free_network_port()}", file] + sys.argv[1:] + cmd, file = generate_ddp_command(world_size, self) # security vulnerability in Snyk scans try: subprocess.run(cmd, check=True) except Exception as e: diff --git a/ultralytics/yolo/engine/validator.py b/ultralytics/yolo/engine/validator.py index e57f3bb..dfd9461 100644 --- a/ultralytics/yolo/engine/validator.py +++ b/ultralytics/yolo/engine/validator.py @@ -119,8 +119,7 @@ class BaseValidator: self.args.workers = 0 # faster CPU val as time dominated by inference, not dataloading if not pt: self.args.rect = False - self.dataloader = self.dataloader or \ - self.get_dataloader(self.data.get("val") or self.data.get("test"), self.args.batch) + self.dataloader = self.dataloader or self.get_dataloader(self.data.get(self.args.split), self.args.batch) model.eval() model.warmup(imgsz=(1 if pt else self.args.batch, 3, imgsz, imgsz)) # warmup diff --git a/ultralytics/yolo/utils/__init__.py b/ultralytics/yolo/utils/__init__.py index 6a9dd2d..de59e9d 100644 --- a/ultralytics/yolo/utils/__init__.py +++ b/ultralytics/yolo/utils/__init__.py @@ -110,6 +110,15 @@ class IterableSimpleNamespace(SimpleNamespace): def __str__(self): return '\n'.join(f"{k}={v}" for k, v in vars(self).items()) + def __getattr__(self, attr): + name = self.__class__.__name__ + raise AttributeError(f""" + '{name}' object has no attribute '{attr}'. This may be caused by a modified or out of date ultralytics + 'default.yaml' file.\nPlease update your code with 'pip install -U ultralytics' and if necessary replace + {DEFAULT_CFG_PATH} with the latest version from + https://github.com/ultralytics/ultralytics/blob/main/ultralytics/yolo/cfg/default.yaml + """) + def get(self, key, default=None): return getattr(self, key, default) @@ -442,6 +451,19 @@ def colorstr(*input): return "".join(colors[x] for x in args) + f"{string}" + colors["end"] +def remove_ansi_codes(string): + """ + Remove ANSI escape sequences from a string. + + Args: + string (str): The input string that may contain ANSI escape sequences. + + Returns: + str: The input string with ANSI escape sequences removed. + """ + return re.sub(r'\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]', '', string) + + def set_logging(name=LOGGING_NAME, verbose=True): # sets up logging for the given name rank = int(os.getenv('RANK', -1)) # rank in world for Multi-GPU trainings diff --git a/ultralytics/yolo/utils/downloads.py b/ultralytics/yolo/utils/downloads.py index 1a5f49e..ff2df8a 100644 --- a/ultralytics/yolo/utils/downloads.py +++ b/ultralytics/yolo/utils/downloads.py @@ -6,7 +6,7 @@ from itertools import repeat from multiprocessing.pool import ThreadPool from pathlib import Path from urllib import parse, request -from zipfile import ZipFile +from zipfile import ZipFile, is_zipfile, BadZipFile import requests import torch @@ -33,6 +33,8 @@ def unzip_file(file, path=None, exclude=('.DS_Store', '__MACOSX')): Unzip a *.zip file to path/, excluding files containing strings in exclude list Replaces: ZipFile(file).extractall(path=path) """ + if not (Path(file).exists() and is_zipfile(file)): + raise BadZipFile(f"File '{file}' does not exist or is a bad zip file.") if path is None: path = Path(file).parent # default path with ZipFile(file) as zipObj: diff --git a/ultralytics/yolo/utils/files.py b/ultralytics/yolo/utils/files.py index 7360ca7..1b78733 100644 --- a/ultralytics/yolo/utils/files.py +++ b/ultralytics/yolo/utils/files.py @@ -6,7 +6,6 @@ import os import urllib from datetime import datetime from pathlib import Path -from zipfile import ZipFile class WorkingDirectory(contextlib.ContextDecorator): @@ -57,16 +56,6 @@ def increment_path(path, exist_ok=False, sep='', mkdir=False): return path -def unzip_file(file, path=None, exclude=('.DS_Store', '__MACOSX')): - # Unzip a *.zip file to path/, excluding files containing strings in exclude list - if path is None: - path = Path(file).parent # default path - with ZipFile(file) as zipObj: - for f in zipObj.namelist(): # list all archived filenames in the zip - if all(x not in f for x in exclude): - zipObj.extract(f, path=path) - - def file_age(path=__file__): # Return days since last file update dt = (datetime.now() - datetime.fromtimestamp(Path(path).stat().st_mtime)) # delta diff --git a/ultralytics/yolo/v8/classify/predict.py b/ultralytics/yolo/v8/classify/predict.py index 1eed2d7..efd311c 100644 --- a/ultralytics/yolo/v8/classify/predict.py +++ b/ultralytics/yolo/v8/classify/predict.py @@ -1,5 +1,4 @@ # Ultralytics YOLO 🚀, GPL-3.0 license -import sys import torch diff --git a/ultralytics/yolo/v8/classify/val.py b/ultralytics/yolo/v8/classify/val.py index 34bbdea..04ec457 100644 --- a/ultralytics/yolo/v8/classify/val.py +++ b/ultralytics/yolo/v8/classify/val.py @@ -1,5 +1,4 @@ # Ultralytics YOLO 🚀, GPL-3.0 license -import sys from ultralytics.yolo.data import build_classification_dataloader from ultralytics.yolo.engine.validator import BaseValidator diff --git a/ultralytics/yolo/v8/detect/predict.py b/ultralytics/yolo/v8/detect/predict.py index 9f5602a..ab7deb3 100644 --- a/ultralytics/yolo/v8/detect/predict.py +++ b/ultralytics/yolo/v8/detect/predict.py @@ -1,5 +1,4 @@ # Ultralytics YOLO 🚀, GPL-3.0 license -import sys import torch diff --git a/ultralytics/yolo/v8/detect/train.py b/ultralytics/yolo/v8/detect/train.py index c199d22..3df4292 100644 --- a/ultralytics/yolo/v8/detect/train.py +++ b/ultralytics/yolo/v8/detect/train.py @@ -1,5 +1,4 @@ # Ultralytics YOLO 🚀, GPL-3.0 license -import sys from copy import copy import torch diff --git a/ultralytics/yolo/v8/detect/val.py b/ultralytics/yolo/v8/detect/val.py index f093b22..b5127b3 100644 --- a/ultralytics/yolo/v8/detect/val.py +++ b/ultralytics/yolo/v8/detect/val.py @@ -1,7 +1,6 @@ # Ultralytics YOLO 🚀, GPL-3.0 license import os -import sys from pathlib import Path import numpy as np @@ -10,8 +9,8 @@ import torch from ultralytics.yolo.data import build_dataloader from ultralytics.yolo.data.dataloaders.v5loader import create_dataloader from ultralytics.yolo.engine.validator import BaseValidator -from ultralytics.yolo.utils import DEFAULT_CFG, colorstr, ops, yaml_load -from ultralytics.yolo.utils.checks import check_file, check_requirements +from ultralytics.yolo.utils import DEFAULT_CFG, colorstr, ops +from ultralytics.yolo.utils.checks import check_requirements from ultralytics.yolo.utils.metrics import ConfusionMatrix, DetMetrics, box_iou from ultralytics.yolo.utils.plotting import output_to_target, plot_images from ultralytics.yolo.utils.torch_utils import de_parallel @@ -42,7 +41,7 @@ class DetectionValidator(BaseValidator): def init_metrics(self, model): head = model.model[-1] if self.training else model.model.model[-1] - val = self.data.get('val', '') # validation path + val = self.data.get(self.args.split, '') # validation path self.is_coco = isinstance(val, str) and val.endswith(f'coco{os.sep}val2017.txt') # is COCO dataset self.class_map = ops.coco80_to_coco91_class() if self.is_coco else list(range(1000)) self.args.save_json |= self.is_coco and not self.training # run on final val if training COCO diff --git a/ultralytics/yolo/v8/segment/predict.py b/ultralytics/yolo/v8/segment/predict.py index e84b03f..4b5cd66 100644 --- a/ultralytics/yolo/v8/segment/predict.py +++ b/ultralytics/yolo/v8/segment/predict.py @@ -1,7 +1,5 @@ # Ultralytics YOLO 🚀, GPL-3.0 license -import sys - import torch from ultralytics.yolo.engine.results import Results diff --git a/ultralytics/yolo/v8/segment/train.py b/ultralytics/yolo/v8/segment/train.py index 61037b8..2d856b0 100644 --- a/ultralytics/yolo/v8/segment/train.py +++ b/ultralytics/yolo/v8/segment/train.py @@ -1,5 +1,4 @@ # Ultralytics YOLO 🚀, GPL-3.0 license -import sys from copy import copy import torch diff --git a/ultralytics/yolo/v8/segment/val.py b/ultralytics/yolo/v8/segment/val.py index fe46b3f..e0d35f8 100644 --- a/ultralytics/yolo/v8/segment/val.py +++ b/ultralytics/yolo/v8/segment/val.py @@ -1,7 +1,6 @@ # Ultralytics YOLO 🚀, GPL-3.0 license import os -import sys from multiprocessing.pool import ThreadPool from pathlib import Path @@ -30,7 +29,7 @@ class SegmentationValidator(DetectionValidator): def init_metrics(self, model): head = model.model[-1] if self.training else model.model.model[-1] - val = self.data.get('val', '') # validation path + val = self.data.get(self.args.split, '') # validation path self.is_coco = isinstance(val, str) and val.endswith(f'coco{os.sep}val2017.txt') # is COCO dataset self.class_map = ops.coco80_to_coco91_class() if self.is_coco else list(range(1000)) self.args.save_json |= self.is_coco and not self.training # run on final val if training COCO