`ultralytics 8.0.24` mosaic, DDP, download fixes (#703)

Co-authored-by: Laughing <61612323+Laughing-q@users.noreply.github.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

mkdocs.yml

@@ -37,10 +37,27 @@ theme:
     - navigation.footer
     - content.tabs.link  # all code tabs change simultaneously
 
-# Version drop-down menu
-# extra:
-#   version:
-#     provider: mike
+# Customization
+copyright: Ultralytics 2023. All rights reserved.
+extra:
+  # version:
+  #   provider: mike  #  version drop-down menu
+  analytics:
+    provider: google
+    property: G-2M5EHKC0BH
+  social:
+    - icon: fontawesome/brands/github
+      link: https://github.com/ultralytics
+    - icon: fontawesome/brands/linkedin
+      link: https://www.linkedin.com/company/ultralytics
+    - icon: fontawesome/brands/twitter
+      link: https://twitter.com/ultralytics
+    - icon: fontawesome/brands/youtube
+      link: https://www.youtube.com/ultralytics
+    - icon: fontawesome/brands/docker
+      link: https://hub.docker.com/r/ultralytics/ultralytics/
+    - icon: fontawesome/brands/python
+      link: https://pypi.org/project/ultralytics/
 
 extra_css:
   - stylesheets/style.css

ultralytics/__init__.py

@@ -1,6 +1,6 @@
 # Ultralytics YOLO 🚀, GPL-3.0 license
 
-__version__ = "8.0.23"
+__version__ = "8.0.24"
 
 from ultralytics.yolo.engine.model import YOLO
 from ultralytics.yolo.utils import ops

ultralytics/hub/utils.py

@@ -100,6 +100,7 @@ def smart_request(*args, retry=3, timeout=30, thread=True, code=-1, method="post
     """
     retry_codes = (408, 500)  # retry only these codes
 
+    @TryExcept(verbose=verbose)
     def func(*func_args, **func_kwargs):
         r = None  # response
         t0 = time.time()  # initial time for timer
@@ -146,7 +147,7 @@ class Traces:
         env = 'Colab' if is_colab() else 'Kaggle' if is_kaggle() else 'Jupyter' if is_jupyter() else \
             'Docker' if is_docker() else platform.system()
         self.rate_limit = 3.0  # rate limit (seconds)
-        self.t = time.time()  # rate limit timer (seconds)
+        self.t = 0.0  # rate limit timer (seconds)
         self.metadata = {
             "sys_argv_name": Path(sys.argv[0]).name,
             "install": 'git' if is_git_dir() else 'pip' if is_pip_package() else 'other',
@@ -159,7 +160,6 @@ class Traces:
                        not is_github_actions_ci() and \
                        (is_pip_package() or get_git_origin_url() == "https://github.com/ultralytics/ultralytics.git")
 
-    @TryExcept(verbose=False)
     def __call__(self, cfg, all_keys=False, traces_sample_rate=1.0):
         """
        Sync traces data if enabled in the global settings

ultralytics/yolo/cfg/__init__.py

@@ -208,8 +208,8 @@ def entrypoint(debug=False):
         elif a in special:
             special[a]()
             return
-        elif a in DEFAULT_CFG_DICT and DEFAULT_CFG_DICT[a] is False:
-            overrides[a] = True  # auto-True for default False args, i.e. 'yolo show' sets show=True
+        elif a in DEFAULT_CFG_DICT and isinstance(DEFAULT_CFG_DICT[a], bool):
+            overrides[a] = True  # auto-True for default bool args, i.e. 'yolo show' sets show=True
         elif a in DEFAULT_CFG_DICT:
             raise SyntaxError(f"'{colorstr('red', 'bold', a)}' is a valid YOLO argument but is missing an '=' sign "
                               f"to set its value, i.e. try '{a}={DEFAULT_CFG_DICT[a]}'\n{CLI_HELP_MSG}")
@@ -262,7 +262,8 @@ def entrypoint(debug=False):
             LOGGER.warning(f"WARNING ⚠️ 'format=' is missing. Using default 'format={overrides['format']}'.")
 
     # Run command in python
-    getattr(model, mode)(**overrides)
+    cfg = get_cfg(overrides=overrides)
+    getattr(model, mode)(**vars(cfg))
 
 
 # Special modes --------------------------------------------------------------------------------------------------------

ultralytics/yolo/data/augment.py

@@ -44,20 +44,8 @@ class Compose:
         self.transforms = transforms
 
     def __call__(self, data):
-        mosaic_p = None
-        mosaic_imgsz = None
-
         for t in self.transforms:
-            if isinstance(t, Mosaic):
-                temp = t(data)
-                mosaic_p = False if temp == data else True
-                mosaic_imgsz = t.imgsz
-                data = temp
-            else:
-                if isinstance(t, RandomPerspective):
-                    t.border = [-mosaic_imgsz // 2, -mosaic_imgsz // 2] if mosaic_p else [0, 0]
-                data = t(data)
-
+            data = t(data)
         return data
 
     def append(self, transform):
@@ -140,7 +128,7 @@ class Mosaic(BaseMixTransform):
             labels_patch = (labels if i == 0 else labels["mix_labels"][i - 1]).copy()
             # Load image
             img = labels_patch["img"]
-            h, w = labels_patch["resized_shape"]
+            h, w = labels_patch.pop("resized_shape")
 
             # place img in img4
             if i == 0:  # top left
@@ -184,11 +172,12 @@ class Mosaic(BaseMixTransform):
             cls.append(labels["cls"])
             instances.append(labels["instances"])
         final_labels = {
+            "im_file": mosaic_labels[0]["im_file"],
             "ori_shape": mosaic_labels[0]["ori_shape"],
             "resized_shape": (self.imgsz * 2, self.imgsz * 2),
-            "im_file": mosaic_labels[0]["im_file"],
             "cls": np.concatenate(cls, 0),
-            "instances": Instances.concatenate(instances, axis=0)}
+            "instances": Instances.concatenate(instances, axis=0),
+            "mosaic_border": self.border}
         final_labels["instances"].clip(self.imgsz * 2, self.imgsz * 2)
         return final_labels
 
@@ -213,7 +202,14 @@ class MixUp(BaseMixTransform):
 
 class RandomPerspective:
 
-    def __init__(self, degrees=0.0, translate=0.1, scale=0.5, shear=0.0, perspective=0.0, border=(0, 0)):
+    def __init__(self,
+                 degrees=0.0,
+                 translate=0.1,
+                 scale=0.5,
+                 shear=0.0,
+                 perspective=0.0,
+                 border=(0, 0),
+                 pre_transform=None):
         self.degrees = degrees
         self.translate = translate
         self.scale = scale
@@ -221,8 +217,9 @@ class RandomPerspective:
         self.perspective = perspective
         # mosaic border
         self.border = border
+        self.pre_transform = pre_transform
 
-    def affine_transform(self, img):
+    def affine_transform(self, img, border):
         # Center
         C = np.eye(3)
 
@@ -255,7 +252,7 @@ class RandomPerspective:
         # Combined rotation matrix
         M = T @ S @ R @ P @ C  # order of operations (right to left) is IMPORTANT
         # affine image
-        if (self.border[0] != 0) or (self.border[1] != 0) or (M != np.eye(3)).any():  # image changed
+        if (border[0] != 0) or (border[1] != 0) or (M != np.eye(3)).any():  # image changed
             if self.perspective:
                 img = cv2.warpPerspective(img, M, dsize=self.size, borderValue=(114, 114, 114))
             else:  # affine
@@ -341,6 +338,10 @@ class RandomPerspective:
         Args:
             labels(Dict): a dict of `bboxes`, `segments`, `keypoints`.
         """
+        if self.pre_transform and "mosaic_border" not in labels:
+            labels = self.pre_transform(labels)
+            labels.pop("ratio_pad")  # do not need ratio pad
+
         img = labels["img"]
         cls = labels["cls"]
         instances = labels.pop("instances")
@@ -348,10 +349,11 @@ class RandomPerspective:
         instances.convert_bbox(format="xyxy")
         instances.denormalize(*img.shape[:2][::-1])
 
-        self.size = img.shape[1] + self.border[1] * 2, img.shape[0] + self.border[0] * 2  # w, h
+        border = labels.pop("mosaic_border", self.border)
+        self.size = img.shape[1] + border[1] * 2, img.shape[0] + border[0] * 2  # w, h
         # M is affine matrix
         # scale for func:`box_candidates`
-        img, M, scale = self.affine_transform(img)
+        img, M, scale = self.affine_transform(img, border)
 
         bboxes = self.apply_bboxes(instances.bboxes, M)
 
@@ -513,8 +515,10 @@ class CopyPaste:
         # Implement Copy-Paste augmentation https://arxiv.org/abs/2012.07177, labels as nx5 np.array(cls, xyxy)
         im = labels["img"]
         cls = labels["cls"]
+        h, w = im.shape[:2]
         instances = labels.pop("instances")
         instances.convert_bbox(format="xyxy")
+        instances.denormalize(w, h)
         if self.p and len(instances.segments):
             n = len(instances)
             _, w, _ = im.shape  # height, width, channels
@@ -605,7 +609,7 @@ class Format:
         self.batch_idx = batch_idx  # keep the batch indexes
 
     def __call__(self, labels):
-        img = labels["img"]
+        img = labels.pop("img")
         h, w = img.shape[:2]
         cls = labels.pop("cls")
         instances = labels.pop("instances")
@@ -654,7 +658,7 @@ class Format:
         return masks, instances, cls
 
 
-def mosaic_transforms(dataset, imgsz, hyp):
+def v8_transforms(dataset, imgsz, hyp):
     pre_transform = Compose([
         Mosaic(dataset, imgsz=imgsz, p=hyp.mosaic, border=[-imgsz // 2, -imgsz // 2]),
         CopyPaste(p=hyp.copy_paste),
@@ -664,7 +668,7 @@ def mosaic_transforms(dataset, imgsz, hyp):
             scale=hyp.scale,
             shear=hyp.shear,
             perspective=hyp.perspective,
-            border=[-imgsz // 2, -imgsz // 2],
+            pre_transform=LetterBox(new_shape=(imgsz, imgsz)),
         ),])
     return Compose([
         pre_transform,
@@ -675,23 +679,6 @@ def mosaic_transforms(dataset, imgsz, hyp):
         RandomFlip(direction="horizontal", p=hyp.fliplr),])  # transforms
 
 
-def affine_transforms(imgsz, hyp):
-    return Compose([
-        LetterBox(new_shape=(imgsz, imgsz)),
-        RandomPerspective(
-            degrees=hyp.degrees,
-            translate=hyp.translate,
-            scale=hyp.scale,
-            shear=hyp.shear,
-            perspective=hyp.perspective,
-            border=[0, 0],
-        ),
-        Albumentations(p=1.0),
-        RandomHSV(hgain=hyp.hsv_h, sgain=hyp.hsv_s, vgain=hyp.hsv_v),
-        RandomFlip(direction="vertical", p=hyp.flipud),
-        RandomFlip(direction="horizontal", p=hyp.fliplr),])  # transforms
-
-
 # Classification augmentations -----------------------------------------------------------------------------------------
 def classify_transforms(size=224):
     # Transforms to apply if albumentations not installed

ultralytics/yolo/data/base.py

@@ -182,6 +182,7 @@ class BaseDataset(Dataset):
 
     def get_label_info(self, index):
         label = self.labels[index].copy()
+        label.pop("shape", None)  # shape is for rect, remove it
         label["img"], label["ori_shape"], label["resized_shape"] = self.load_image(index)
         label["ratio_pad"] = (
             label["resized_shape"][0] / label["ori_shape"][0],

ultralytics/yolo/data/dataset.py

@@ -136,8 +136,9 @@ class YOLODataset(BaseDataset):
     # TODO: use hyp config to set all these augmentations
     def build_transforms(self, hyp=None):
         if self.augment:
-            mosaic = self.augment and not self.rect
-            transforms = mosaic_transforms(self, self.imgsz, hyp) if mosaic else affine_transforms(self.imgsz, hyp)
+            hyp.mosaic = hyp.mosaic if self.augment and not self.rect else 0.0
+            hyp.mixup = hyp.mixup if self.augment and not self.rect else 0.0
+            transforms = v8_transforms(self, self.imgsz, hyp)
         else:
             transforms = Compose([LetterBox(new_shape=(self.imgsz, self.imgsz), scaleup=False)])
         transforms.append(
@@ -151,15 +152,10 @@ class YOLODataset(BaseDataset):
         return transforms
 
     def close_mosaic(self, hyp):
-        self.transforms = affine_transforms(self.imgsz, hyp)
-        self.transforms.append(
-            Format(bbox_format="xywh",
-                   normalize=True,
-                   return_mask=self.use_segments,
-                   return_keypoint=self.use_keypoints,
-                   batch_idx=True,
-                   mask_ratio=hyp.mask_ratio,
-                   mask_overlap=hyp.overlap_mask))
+        hyp.mosaic = 0.0  # set mosaic ratio=0.0
+        hyp.copy_paste = 0.0  # keep the same behavior as previous v8 close-mosaic
+        hyp.mixup = 0.0  # keep the same behavior as previous v8 close-mosaic
+        self.transforms = self.build_transforms(hyp)
 
     def update_labels_info(self, label):
         """custom your label format here"""
@@ -175,8 +171,6 @@ class YOLODataset(BaseDataset):
 
     @staticmethod
     def collate_fn(batch):
-        # TODO: returning a dict can make thing easier and cleaner when using dataset in training
-        # but I don't know if this will slow down a little bit.
         new_batch = {}
         keys = batch[0].keys()
         values = list(zip(*[list(b.values()) for b in batch]))

ultralytics/yolo/data/utils.py

@@ -246,7 +246,7 @@ def check_det_dataset(dataset, autodownload=True):
                 r = exec(s, {'yaml': data})  # return None
             dt = f'({round(time.time() - t, 1)}s)'
             s = f"success ✅ {dt}, saved to {colorstr('bold', DATASETS_DIR)}" if r in (0, None) else f"failure {dt} ❌"
-            LOGGER.info(f"Dataset download {s}")
+            LOGGER.info(f"Dataset download {s}\n")
     check_font('Arial.ttf' if is_ascii(data['names']) else 'Arial.Unicode.ttf')  # download fonts
 
     return data  # dictionary

ultralytics/yolo/engine/model.py

@@ -7,7 +7,7 @@ from ultralytics.nn.tasks import (ClassificationModel, DetectionModel, Segmentat
                                   guess_model_task)
 from ultralytics.yolo.cfg import get_cfg
 from ultralytics.yolo.engine.exporter import Exporter
-from ultralytics.yolo.utils import DEFAULT_CFG, LOGGER, callbacks, yaml_load
+from ultralytics.yolo.utils import DEFAULT_CFG, LOGGER, RANK, callbacks, yaml_load
 from ultralytics.yolo.utils.checks import check_yaml
 from ultralytics.yolo.utils.torch_utils import smart_inference_mode
 
@@ -205,8 +205,9 @@ class YOLO:
             self.model = self.trainer.model
         self.trainer.train()
         # update model and cfg after training
-        self.model, _ = attempt_load_one_weight(str(self.trainer.best))
-        self.overrides = self.model.args
+        if RANK in {0, -1}:
+            self.model, _ = attempt_load_one_weight(str(self.trainer.best))
+            self.overrides = self.model.args
 
     def to(self, device):
         """

ultralytics/yolo/engine/predictor.py

@@ -135,6 +135,8 @@ class BasePredictor:
 
     def stream_inference(self, source=None, model=None):
         self.run_callbacks("on_predict_start")
+        if self.args.verbose:
+            LOGGER.info("")
 
         # setup model
         if not self.model:

ultralytics/yolo/engine/trainer.py

@@ -518,7 +518,7 @@ class BaseTrainer:
             last = Path(check_file(resume) if isinstance(resume, (str, Path)) else get_latest_run())
             args_yaml = last.parent.parent / 'args.yaml'  # train options yaml
             assert args_yaml.is_file(), \
-                FileNotFoundError('Resume checkpoint f{last} not found. '
+                FileNotFoundError(f'Resume checkpoint {last} not found. '
                                   'Please pass a valid checkpoint to resume from, i.e. yolo resume=path/to/last.pt')
             args = get_cfg(args_yaml)  # replace
             args.model, resume = str(last), True  # reinstate

ultralytics/yolo/utils/checks.py

@@ -93,8 +93,7 @@ def check_version(current: str = "0.0.0",
     Returns:
         bool: True if minimum version is met, False otherwise.
     """
-    from pkg_resources import parse_version
-    current, minimum = (parse_version(x) for x in (current, minimum))
+    current, minimum = (pkg.parse_version(x) for x in (current, minimum))
     result = (current == minimum) if pinned else (current >= minimum)  # bool
     warning_message = f"WARNING ⚠️ {name}{minimum} is required by YOLOv8, but {name}{current} is currently installed"
     if hard:

ultralytics/yolo/utils/downloads.py

@@ -1,29 +1,31 @@
 # Ultralytics YOLO 🚀, GPL-3.0 license
 
 import contextlib
-import os
 import subprocess
-import urllib
 from itertools import repeat
 from multiprocessing.pool import ThreadPool
 from pathlib import Path
+from urllib import parse, request
 from zipfile import ZipFile
 
 import requests
 import torch
+from tqdm import tqdm
 
 from ultralytics.yolo.utils import LOGGER
 
 
 def is_url(url, check=True):
     # Check if string is URL and check if URL exists
-    try:
+    with contextlib.suppress(Exception):
         url = str(url)
-        result = urllib.parse.urlparse(url)
+        result = parse.urlparse(url)
         assert all([result.scheme, result.netloc])  # check if is url
-        return (urllib.request.urlopen(url).getcode() == 200) if check else True  # check if exists online
-    except (AssertionError, urllib.request.HTTPError):
-        return False
+        if check:
+            with request.urlopen(url) as response:
+                return response.getcode() == 200  # check if exists online
+        return True
+    return False
 
 
 def safe_download(url,
@@ -57,35 +59,50 @@ def safe_download(url,
     else:  # does not exist
         assert dir or file, 'dir or file required for download'
         f = dir / Path(url).name if dir else Path(file)
-        LOGGER.info(f'Downloading {url} to {f}...')
+        desc = f'Downloading {url} to {f}'
+        LOGGER.info(f'{desc}...')
         f.parent.mkdir(parents=True, exist_ok=True)  # make directory if missing
         for i in range(retry + 1):
             try:
                 if curl or i > 0:  # curl download with retry, continue
                     s = 'sS' * (not progress)  # silent
-                    r = os.system(f'curl -# -{s}L "{url}" -o "{f}" --retry 9 -C -')
-                else:  # torch download
-                    r = torch.hub.download_url_to_file(url, f, progress=progress)
-                assert r in {0, None}
+                    r = subprocess.run(['curl', '-#', f'-{s}L', url, '-o', f, '--retry', '9', '-C', '-']).returncode
+                    assert r == 0, f'Curl return value {r}'
+                else:  # urllib download
+                    method = 'torch'
+                    if method == 'torch':
+                        torch.hub.download_url_to_file(url, f, progress=progress)
+                    else:
+                        from ultralytics.yolo.utils import TQDM_BAR_FORMAT
+                        with request.urlopen(url) as response, tqdm(total=int(response.getheader("Content-Length", 0)),
+                                                                    desc=desc,
+                                                                    disable=not progress,
+                                                                    unit='B',
+                                                                    unit_scale=True,
+                                                                    unit_divisor=1024,
+                                                                    bar_format=TQDM_BAR_FORMAT) as pbar:
+                            with open(f, "wb") as f_opened:
+                                for data in response:
+                                    f_opened.write(data)
+                                    pbar.update(len(data))
+
+                if f.exists():
+                    if f.stat().st_size > min_bytes:
+                        break  # success
+                    f.unlink()  # remove partial downloads
             except Exception as e:
                 if i >= retry:
                     raise ConnectionError(f'❌  Download failure for {url}') from e
                 LOGGER.warning(f'⚠️ Download failure, retrying {i + 1}/{retry} {url}...')
-                continue
-
-            if f.exists():
-                if f.stat().st_size > min_bytes:
-                    break  # success
-                f.unlink()  # remove partial downloads
 
     if unzip and f.exists() and f.suffix in {'.zip', '.tar', '.gz'}:
         LOGGER.info(f'Unzipping {f}...')
         if f.suffix == '.zip':
             ZipFile(f).extractall(path=f.parent)  # unzip
         elif f.suffix == '.tar':
-            os.system(f'tar xf {f} --directory {f.parent}')  # unzip
+            subprocess.run(['tar', 'xf', f, '--directory', f.parent], check=True)  # unzip
         elif f.suffix == '.gz':
-            os.system(f'tar xfz {f} --directory {f.parent}')  # unzip
+            subprocess.run(['tar', 'xfz', f, '--directory', f.parent], check=True)  # unzip
         if delete:
             f.unlink()  # remove zip
 
@@ -95,7 +112,6 @@ def attempt_download_asset(file, repo='ultralytics/assets', release='v0.0.0'):
     from ultralytics.yolo.utils import SETTINGS
 
     def github_assets(repository, version='latest'):
-        # Return GitHub repo tag and assets (i.e. ['yolov8n.pt', 'yolov5m.pt', ...])
         # Return GitHub repo tag and assets (i.e. ['yolov8n.pt', 'yolov8s.pt', ...])
         if version != 'latest':
             version = f'tags/{version}'  # i.e. tags/v6.2
@@ -109,7 +125,7 @@ def attempt_download_asset(file, repo='ultralytics/assets', release='v0.0.0'):
         return str(SETTINGS['weights_dir'] / file)
     else:
         # URL specified
-        name = Path(urllib.parse.unquote(str(file))).name  # decode '%2F' to '/' etc.
+        name = Path(parse.unquote(str(file))).name  # decode '%2F' to '/' etc.
         if str(file).startswith(('http:/', 'https:/')):  # download
             url = str(file).replace(':/', '://')  # Pathlib turns :// -> :/
             file = name.split('?')[0]  # parse authentication https://url.com/file.txt?auth...
@@ -128,7 +144,7 @@ def attempt_download_asset(file, repo='ultralytics/assets', release='v0.0.0'):
                 tag, assets = github_assets(repo)  # latest release
             except Exception:
                 try:
-                    tag = subprocess.check_output('git tag', shell=True, stderr=subprocess.STDOUT).decode().split()[-1]
+                    tag = subprocess.check_output(["git", "tag"]).decode().split()[-1]
                 except Exception:
                     tag = release
 

ultralytics/yolo/utils/plotting.py

@@ -10,10 +10,11 @@ import numpy as np
 import pandas as pd
 import torch
 from PIL import Image, ImageDraw, ImageFont
+from PIL import __version__ as pil_version
 
 from ultralytics.yolo.utils import threaded
 
-from .checks import check_font, is_ascii
+from .checks import check_font, check_version, is_ascii
 from .files import increment_path
 from .ops import clip_coords, scale_image, xywh2xyxy, xyxy2xywh
 
@@ -46,6 +47,7 @@ class Annotator:
         non_ascii = not is_ascii(example)  # non-latin labels, i.e. asian, arabic, cyrillic
         self.pil = pil or non_ascii
         if self.pil:  # use PIL
+            self.pil_9_2_0_check = check_version(pil_version, '9.2.0')  # deprecation check
             self.im = im if isinstance(im, Image.Image) else Image.fromarray(im)
             self.draw = ImageDraw.Draw(self.im)
             try:
@@ -65,8 +67,10 @@ class Annotator:
         if self.pil or not is_ascii(label):
             self.draw.rectangle(box, width=self.lw, outline=color)  # box
             if label:
-                w, h = self.font.getsize(label)  # text width, height (WARNING: deprecated) in 9.2.0
-                # _, _, w, h = self.font.getbbox(label)  # text width, height (New)
+                if self.pil_9_2_0_check:
+                    _, _, w, h = self.font.getbbox(label)  # text width, height (New)
+                else:
+                    w, h = self.font.getsize(label)  # text width, height (Old, deprecated in 9.2.0)
                 outside = box[1] - h >= 0  # label fits outside box
                 self.draw.rectangle(
                     (box[0], box[1] - h if outside else box[1], box[0] + w + 1,

ultralytics/yolo/utils/torch_utils.py

@@ -58,7 +58,7 @@ def DDP_model(model):
 def select_device(device='', batch=0, newline=False):
     # device = None or 'cpu' or 0 or '0' or '0,1,2,3'
     from ultralytics import __version__
-    s = f'Ultralytics YOLOv{__version__} 🚀 Python-{platform.python_version()} torch-{torch.__version__} '
+    s = f"Ultralytics YOLOv{__version__} 🚀 Python-{platform.python_version()} torch-{torch.__version__} "
     device = str(device).lower()
     for remove in 'cuda:', 'none', '(', ')', '[', ']', "'", ' ':
         device = device.replace(remove, '')  # to string, 'cuda:0' -> '0' and '(0, 1)' -> '0,1'