You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
103 lines
3.3 KiB
103 lines
3.3 KiB
import contextlib
|
|
import glob
|
|
import os
|
|
import urllib
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from zipfile import ZipFile
|
|
|
|
import yaml
|
|
|
|
|
|
class WorkingDirectory(contextlib.ContextDecorator):
|
|
# Usage: @WorkingDirectory(dir) decorator or 'with WorkingDirectory(dir):' context manager
|
|
def __init__(self, new_dir):
|
|
self.dir = new_dir # new dir
|
|
self.cwd = Path.cwd().resolve() # current dir
|
|
|
|
def __enter__(self):
|
|
os.chdir(self.dir)
|
|
|
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
os.chdir(self.cwd)
|
|
|
|
|
|
def increment_path(path, exist_ok=False, sep='', mkdir=False):
|
|
"""
|
|
Increment file or directory path, i.e. runs/exp --> runs/exp{sep}2, runs/exp{sep}3, ... etc.
|
|
# TODO: docs
|
|
"""
|
|
path = Path(path) # os-agnostic
|
|
if path.exists() and not exist_ok:
|
|
path, suffix = (path.with_suffix(''), path.suffix) if path.is_file() else (path, '')
|
|
|
|
# Method 1
|
|
for n in range(2, 9999):
|
|
p = f'{path}{sep}{n}{suffix}' # increment path
|
|
if not os.path.exists(p): #
|
|
break
|
|
path = Path(p)
|
|
|
|
if mkdir:
|
|
path.mkdir(parents=True, exist_ok=True) # make directory
|
|
|
|
return path
|
|
|
|
|
|
def yaml_save(file='data.yaml', data=None):
|
|
# Single-line safe yaml saving
|
|
with open(file, 'w') as f:
|
|
yaml.safe_dump({k: str(v) if isinstance(v, Path) else v for k, v in data.items()}, f, sort_keys=False)
|
|
|
|
|
|
def yaml_load(file='data.yaml'):
|
|
# Single-line safe yaml loading
|
|
with open(file, errors='ignore') as f:
|
|
return {**yaml.safe_load(f), 'yaml_file': file} # add YAML filename to dict and return
|
|
|
|
|
|
def unzip_file(file, path=None, exclude=('.DS_Store', '__MACOSX')):
|
|
# Unzip a *.zip file to path/, excluding files containing strings in exclude list
|
|
if path is None:
|
|
path = Path(file).parent # default path
|
|
with ZipFile(file) as zipObj:
|
|
for f in zipObj.namelist(): # list all archived filenames in the zip
|
|
if all(x not in f for x in exclude):
|
|
zipObj.extract(f, path=path)
|
|
|
|
|
|
def file_age(path=__file__):
|
|
# Return days since last file update
|
|
dt = (datetime.now() - datetime.fromtimestamp(Path(path).stat().st_mtime)) # delta
|
|
return dt.days # + dt.seconds / 86400 # fractional days
|
|
|
|
|
|
def file_date(path=__file__):
|
|
# Return human-readable file modification date, i.e. '2021-3-26'
|
|
t = datetime.fromtimestamp(Path(path).stat().st_mtime)
|
|
return f'{t.year}-{t.month}-{t.day}'
|
|
|
|
|
|
def file_size(path):
|
|
# Return file/dir size (MB)
|
|
mb = 1 << 20 # bytes to MiB (1024 ** 2)
|
|
path = Path(path)
|
|
if path.is_file():
|
|
return path.stat().st_size / mb
|
|
elif path.is_dir():
|
|
return sum(f.stat().st_size for f in path.glob('**/*') if f.is_file()) / mb
|
|
else:
|
|
return 0.0
|
|
|
|
|
|
def url2file(url):
|
|
# Convert URL to filename, i.e. https://url.com/file.txt?auth -> file.txt
|
|
url = str(Path(url)).replace(':/', '://') # Pathlib turns :// -> :/
|
|
return Path(urllib.parse.unquote(url)).name.split('?')[0] # '%2F' to '/', split https://url.com/file.txt?auth
|
|
|
|
|
|
def get_latest_run(search_dir='.'):
|
|
# Return path to most recent 'last.pt' in /runs (i.e. to --resume from)
|
|
last_list = glob.glob(f'{search_dir}/**/last*.pt', recursive=True)
|
|
return max(last_list, key=os.path.getctime) if last_list else ''
|