|
|
|
|
@ -43,16 +43,18 @@ def bbox_ioa(box1, box2, eps=1e-7):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def box_iou(box1, box2, eps=1e-7):
|
|
|
|
|
# https://github.com/pytorch/vision/blob/master/torchvision/ops/boxes.py
|
|
|
|
|
"""
|
|
|
|
|
Return intersection-over-union (Jaccard index) of boxes.
|
|
|
|
|
Both sets of boxes are expected to be in (x1, y1, x2, y2) format.
|
|
|
|
|
Based on https://github.com/pytorch/vision/blob/master/torchvision/ops/boxes.py
|
|
|
|
|
|
|
|
|
|
Arguments:
|
|
|
|
|
box1 (Tensor[N, 4])
|
|
|
|
|
box2 (Tensor[M, 4])
|
|
|
|
|
eps
|
|
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
|
iou (Tensor[N, M]): the NxM matrix containing the pairwise
|
|
|
|
|
IoU values for every element in boxes1 and boxes2
|
|
|
|
|
iou (Tensor[N, M]): the NxM matrix containing the pairwise IoU values for every element in boxes1 and boxes2
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
# inter(N,M) = (rb(N,M,2) - lt(N,M,2)).clamp(0).prod(2)
|
|
|
|
|
@ -109,7 +111,7 @@ def mask_iou(mask1, mask2, eps=1e-7):
|
|
|
|
|
mask1: [N, n] m1 means number of predicted objects
|
|
|
|
|
mask2: [M, n] m2 means number of gt objects
|
|
|
|
|
Note: n means image_w x image_h
|
|
|
|
|
return: masks iou, [N, M]
|
|
|
|
|
Returns: masks iou, [N, M]
|
|
|
|
|
"""
|
|
|
|
|
intersection = torch.matmul(mask1, mask2.t()).clamp(0)
|
|
|
|
|
union = (mask1.sum(1)[:, None] + mask2.sum(1)[None]) - intersection # (area1 + area2) - intersection
|
|
|
|
|
@ -121,7 +123,7 @@ def masks_iou(mask1, mask2, eps=1e-7):
|
|
|
|
|
mask1: [N, n] m1 means number of predicted objects
|
|
|
|
|
mask2: [N, n] m2 means number of gt objects
|
|
|
|
|
Note: n means image_w x image_h
|
|
|
|
|
return: masks iou, (N, )
|
|
|
|
|
Returns: masks iou, (N, )
|
|
|
|
|
"""
|
|
|
|
|
intersection = (mask1 * mask2).sum(1).clamp(0) # (N, )
|
|
|
|
|
union = (mask1.sum(1) + mask2.sum(1))[None] - intersection # (area1 + area2) - intersection
|
|
|
|
|
@ -317,10 +319,10 @@ def plot_mc_curve(px, py, save_dir=Path('mc_curve.png'), names=(), xlabel='Confi
|
|
|
|
|
|
|
|
|
|
def compute_ap(recall, precision):
|
|
|
|
|
""" Compute the average precision, given the recall and precision curves
|
|
|
|
|
# Arguments
|
|
|
|
|
Arguments:
|
|
|
|
|
recall: The recall curve (list)
|
|
|
|
|
precision: The precision curve (list)
|
|
|
|
|
# Returns
|
|
|
|
|
Returns:
|
|
|
|
|
Average precision, precision curve, recall curve
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
@ -344,17 +346,30 @@ def compute_ap(recall, precision):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def ap_per_class(tp, conf, pred_cls, target_cls, plot=False, save_dir=Path(), names=(), eps=1e-16, prefix=''):
|
|
|
|
|
""" Compute the average precision, given the recall and precision curves.
|
|
|
|
|
Source: https://github.com/rafaelpadilla/Object-Detection-Metrics.
|
|
|
|
|
# Arguments
|
|
|
|
|
tp: True positives (nparray, nx1 or nx10).
|
|
|
|
|
conf: Objectness value from 0-1 (nparray).
|
|
|
|
|
pred_cls: Predicted object classes (nparray).
|
|
|
|
|
target_cls: True object classes (nparray).
|
|
|
|
|
plot: Plot precision-recall curve at mAP@0.5
|
|
|
|
|
save_dir: Plot save directory
|
|
|
|
|
# Returns
|
|
|
|
|
The average precision as computed in py-faster-rcnn.
|
|
|
|
|
"""
|
|
|
|
|
Computes the average precision per class for object detection evaluation.
|
|
|
|
|
|
|
|
|
|
Args:
|
|
|
|
|
tp (np.ndarray): Binary array indicating whether the detection is correct (True) or not (False).
|
|
|
|
|
conf (np.ndarray): Array of confidence scores of the detections.
|
|
|
|
|
pred_cls (np.ndarray): Array of predicted classes of the detections.
|
|
|
|
|
target_cls (np.ndarray): Array of true classes of the detections.
|
|
|
|
|
plot (bool, optional): Whether to plot PR curves or not. Defaults to False.
|
|
|
|
|
save_dir (Path, optional): Directory to save the PR curves. Defaults to an empty path.
|
|
|
|
|
names (tuple, optional): Tuple of class names to plot PR curves. Defaults to an empty tuple.
|
|
|
|
|
eps (float, optional): A small value to avoid division by zero. Defaults to 1e-16.
|
|
|
|
|
prefix (str, optional): A prefix string for saving the plot files. Defaults to an empty string.
|
|
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
|
(tuple): A tuple of six arrays and one array of unique classes, where:
|
|
|
|
|
tp (np.ndarray): True positive counts for each class.
|
|
|
|
|
fp (np.ndarray): False positive counts for each class.
|
|
|
|
|
p (np.ndarray): Precision values at each confidence threshold.
|
|
|
|
|
r (np.ndarray): Recall values at each confidence threshold.
|
|
|
|
|
f1 (np.ndarray): F1-score values at each confidence threshold.
|
|
|
|
|
ap (np.ndarray): Average precision for each class at different IoU thresholds.
|
|
|
|
|
unique_classes (np.ndarray): An array of unique classes that have data.
|
|
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
# Sort by objectness
|
|
|
|
|
@ -411,6 +426,32 @@ def ap_per_class(tp, conf, pred_cls, target_cls, plot=False, save_dir=Path(), na
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Metric:
|
|
|
|
|
"""
|
|
|
|
|
Class for computing evaluation metrics for YOLOv8 model.
|
|
|
|
|
|
|
|
|
|
Attributes:
|
|
|
|
|
p (list): Precision for each class. Shape: (nc,).
|
|
|
|
|
r (list): Recall for each class. Shape: (nc,).
|
|
|
|
|
f1 (list): F1 score for each class. Shape: (nc,).
|
|
|
|
|
all_ap (list): AP scores for all classes and all IoU thresholds. Shape: (nc, 10).
|
|
|
|
|
ap_class_index (list): Index of class for each AP score. Shape: (nc,).
|
|
|
|
|
nc (int): Number of classes.
|
|
|
|
|
|
|
|
|
|
Methods:
|
|
|
|
|
ap50(): AP at IoU threshold of 0.5 for all classes. Returns: List of AP scores. Shape: (nc,) or [].
|
|
|
|
|
ap(): AP at IoU thresholds from 0.5 to 0.95 for all classes. Returns: List of AP scores. Shape: (nc,) or [].
|
|
|
|
|
mp(): Mean precision of all classes. Returns: Float.
|
|
|
|
|
mr(): Mean recall of all classes. Returns: Float.
|
|
|
|
|
map50(): Mean AP at IoU threshold of 0.5 for all classes. Returns: Float.
|
|
|
|
|
map75(): Mean AP at IoU threshold of 0.75 for all classes. Returns: Float.
|
|
|
|
|
map(): Mean AP at IoU thresholds from 0.5 to 0.95 for all classes. Returns: Float.
|
|
|
|
|
mean_results(): Mean of results, returns mp, mr, map50, map.
|
|
|
|
|
class_result(i): Class-aware result, returns p[i], r[i], ap50[i], ap[i].
|
|
|
|
|
maps(): mAP of each class. Returns: Array of mAP scores, shape: (nc,).
|
|
|
|
|
fitness(): Model fitness as a weighted combination of metrics. Returns: Float.
|
|
|
|
|
update(results): Update metric attributes with new evaluation results.
|
|
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
def __init__(self) -> None:
|
|
|
|
|
self.p = [] # (nc, )
|
|
|
|
|
@ -420,10 +461,14 @@ class Metric:
|
|
|
|
|
self.ap_class_index = [] # (nc, )
|
|
|
|
|
self.nc = 0
|
|
|
|
|
|
|
|
|
|
def __getattr__(self, attr):
|
|
|
|
|
name = self.__class__.__name__
|
|
|
|
|
raise AttributeError(f"'{name}' object has no attribute '{attr}'. See valid attributes below.\n{self.__doc__}")
|
|
|
|
|
|
|
|
|
|
@property
|
|
|
|
|
def ap50(self):
|
|
|
|
|
"""AP@0.5 of all classes.
|
|
|
|
|
Return:
|
|
|
|
|
Returns:
|
|
|
|
|
(nc, ) or [].
|
|
|
|
|
"""
|
|
|
|
|
return self.all_ap[:, 0] if len(self.all_ap) else []
|
|
|
|
|
@ -431,7 +476,7 @@ class Metric:
|
|
|
|
|
@property
|
|
|
|
|
def ap(self):
|
|
|
|
|
"""AP@0.5:0.95
|
|
|
|
|
Return:
|
|
|
|
|
Returns:
|
|
|
|
|
(nc, ) or [].
|
|
|
|
|
"""
|
|
|
|
|
return self.all_ap.mean(1) if len(self.all_ap) else []
|
|
|
|
|
@ -439,7 +484,7 @@ class Metric:
|
|
|
|
|
@property
|
|
|
|
|
def mp(self):
|
|
|
|
|
"""mean precision of all classes.
|
|
|
|
|
Return:
|
|
|
|
|
Returns:
|
|
|
|
|
float.
|
|
|
|
|
"""
|
|
|
|
|
return self.p.mean() if len(self.p) else 0.0
|
|
|
|
|
@ -447,7 +492,7 @@ class Metric:
|
|
|
|
|
@property
|
|
|
|
|
def mr(self):
|
|
|
|
|
"""mean recall of all classes.
|
|
|
|
|
Return:
|
|
|
|
|
Returns:
|
|
|
|
|
float.
|
|
|
|
|
"""
|
|
|
|
|
return self.r.mean() if len(self.r) else 0.0
|
|
|
|
|
@ -455,7 +500,7 @@ class Metric:
|
|
|
|
|
@property
|
|
|
|
|
def map50(self):
|
|
|
|
|
"""Mean AP@0.5 of all classes.
|
|
|
|
|
Return:
|
|
|
|
|
Returns:
|
|
|
|
|
float.
|
|
|
|
|
"""
|
|
|
|
|
return self.all_ap[:, 0].mean() if len(self.all_ap) else 0.0
|
|
|
|
|
@ -463,7 +508,7 @@ class Metric:
|
|
|
|
|
@property
|
|
|
|
|
def map75(self):
|
|
|
|
|
"""Mean AP@0.75 of all classes.
|
|
|
|
|
Return:
|
|
|
|
|
Returns:
|
|
|
|
|
float.
|
|
|
|
|
"""
|
|
|
|
|
return self.all_ap[:, 5].mean() if len(self.all_ap) else 0.0
|
|
|
|
|
@ -471,7 +516,7 @@ class Metric:
|
|
|
|
|
@property
|
|
|
|
|
def map(self):
|
|
|
|
|
"""Mean AP@0.5:0.95 of all classes.
|
|
|
|
|
Return:
|
|
|
|
|
Returns:
|
|
|
|
|
float.
|
|
|
|
|
"""
|
|
|
|
|
return self.all_ap.mean() if len(self.all_ap) else 0.0
|
|
|
|
|
@ -506,6 +551,32 @@ class Metric:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class DetMetrics:
|
|
|
|
|
"""
|
|
|
|
|
This class is a utility class for computing detection metrics such as precision, recall, and mean average precision
|
|
|
|
|
(mAP) of an object detection model.
|
|
|
|
|
|
|
|
|
|
Args:
|
|
|
|
|
save_dir (Path): A path to the directory where the output plots will be saved. Defaults to current directory.
|
|
|
|
|
plot (bool): A flag that indicates whether to plot precision-recall curves for each class. Defaults to False.
|
|
|
|
|
names (tuple of str): A tuple of strings that represents the names of the classes. Defaults to an empty tuple.
|
|
|
|
|
|
|
|
|
|
Attributes:
|
|
|
|
|
save_dir (Path): A path to the directory where the output plots will be saved.
|
|
|
|
|
plot (bool): A flag that indicates whether to plot the precision-recall curves for each class.
|
|
|
|
|
names (tuple of str): A tuple of strings that represents the names of the classes.
|
|
|
|
|
box (Metric): An instance of the Metric class for storing the results of the detection metrics.
|
|
|
|
|
speed (dict): A dictionary for storing the execution time of different parts of the detection process.
|
|
|
|
|
|
|
|
|
|
Methods:
|
|
|
|
|
process(tp, conf, pred_cls, target_cls): Updates the metric results with the latest batch of predictions.
|
|
|
|
|
keys: Returns a list of keys for accessing the computed detection metrics.
|
|
|
|
|
mean_results: Returns a list of mean values for the computed detection metrics.
|
|
|
|
|
class_result(i): Returns a list of values for the computed detection metrics for a specific class.
|
|
|
|
|
maps: Returns a dictionary of mean average precision (mAP) values for different IoU thresholds.
|
|
|
|
|
fitness: Computes the fitness score based on the computed detection metrics.
|
|
|
|
|
ap_class_index: Returns a list of class indices sorted by their average precision (AP) values.
|
|
|
|
|
results_dict: Returns a dictionary that maps detection metric keys to their computed values.
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
def __init__(self, save_dir=Path('.'), plot=False, names=()) -> None:
|
|
|
|
|
self.save_dir = save_dir
|
|
|
|
|
@ -514,6 +585,10 @@ class DetMetrics:
|
|
|
|
|
self.box = Metric()
|
|
|
|
|
self.speed = {'preprocess': 0.0, 'inference': 0.0, 'loss': 0.0, 'postprocess': 0.0}
|
|
|
|
|
|
|
|
|
|
def __getattr__(self, attr):
|
|
|
|
|
name = self.__class__.__name__
|
|
|
|
|
raise AttributeError(f"'{name}' object has no attribute '{attr}'. See valid attributes below.\n{self.__doc__}")
|
|
|
|
|
|
|
|
|
|
def process(self, tp, conf, pred_cls, target_cls):
|
|
|
|
|
results = ap_per_class(tp, conf, pred_cls, target_cls, plot=self.plot, save_dir=self.save_dir,
|
|
|
|
|
names=self.names)[2:]
|
|
|
|
|
@ -548,6 +623,31 @@ class DetMetrics:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class SegmentMetrics:
|
|
|
|
|
"""
|
|
|
|
|
Calculates and aggregates detection and segmentation metrics over a given set of classes.
|
|
|
|
|
|
|
|
|
|
Args:
|
|
|
|
|
save_dir (Path): Path to the directory where the output plots should be saved. Default is the current directory.
|
|
|
|
|
plot (bool): Whether to save the detection and segmentation plots. Default is False.
|
|
|
|
|
names (list): List of class names. Default is an empty list.
|
|
|
|
|
|
|
|
|
|
Attributes:
|
|
|
|
|
save_dir (Path): Path to the directory where the output plots should be saved.
|
|
|
|
|
plot (bool): Whether to save the detection and segmentation plots.
|
|
|
|
|
names (list): List of class names.
|
|
|
|
|
box (Metric): An instance of the Metric class to calculate box detection metrics.
|
|
|
|
|
seg (Metric): An instance of the Metric class to calculate mask segmentation metrics.
|
|
|
|
|
speed (dict): Dictionary to store the time taken in different phases of inference.
|
|
|
|
|
|
|
|
|
|
Methods:
|
|
|
|
|
process(tp_m, tp_b, conf, pred_cls, target_cls): Processes metrics over the given set of predictions.
|
|
|
|
|
mean_results(): Returns the mean of the detection and segmentation metrics over all the classes.
|
|
|
|
|
class_result(i): Returns the detection and segmentation metrics of class `i`.
|
|
|
|
|
maps: Returns the mean Average Precision (mAP) scores for IoU thresholds ranging from 0.50 to 0.95.
|
|
|
|
|
fitness: Returns the fitness scores, which are a single weighted combination of metrics.
|
|
|
|
|
ap_class_index: Returns the list of indices of classes used to compute Average Precision (AP).
|
|
|
|
|
results_dict: Returns the dictionary containing all the detection and segmentation metrics and fitness score.
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
def __init__(self, save_dir=Path('.'), plot=False, names=()) -> None:
|
|
|
|
|
self.save_dir = save_dir
|
|
|
|
|
@ -557,7 +657,22 @@ class SegmentMetrics:
|
|
|
|
|
self.seg = Metric()
|
|
|
|
|
self.speed = {'preprocess': 0.0, 'inference': 0.0, 'loss': 0.0, 'postprocess': 0.0}
|
|
|
|
|
|
|
|
|
|
def __getattr__(self, attr):
|
|
|
|
|
name = self.__class__.__name__
|
|
|
|
|
raise AttributeError(f"'{name}' object has no attribute '{attr}'. See valid attributes below.\n{self.__doc__}")
|
|
|
|
|
|
|
|
|
|
def process(self, tp_m, tp_b, conf, pred_cls, target_cls):
|
|
|
|
|
"""
|
|
|
|
|
Processes the detection and segmentation metrics over the given set of predictions.
|
|
|
|
|
|
|
|
|
|
Args:
|
|
|
|
|
tp_m (list): List of True Positive masks.
|
|
|
|
|
tp_b (list): List of True Positive boxes.
|
|
|
|
|
conf (list): List of confidence scores.
|
|
|
|
|
pred_cls (list): List of predicted classes.
|
|
|
|
|
target_cls (list): List of target classes.
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
results_mask = ap_per_class(tp_m,
|
|
|
|
|
conf,
|
|
|
|
|
pred_cls,
|
|
|
|
|
@ -610,12 +725,32 @@ class SegmentMetrics:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class ClassifyMetrics:
|
|
|
|
|
"""
|
|
|
|
|
Class for computing classification metrics including top-1 and top-5 accuracy.
|
|
|
|
|
|
|
|
|
|
Attributes:
|
|
|
|
|
top1 (float): The top-1 accuracy.
|
|
|
|
|
top5 (float): The top-5 accuracy.
|
|
|
|
|
speed (Dict[str, float]): A dictionary containing the time taken for each step in the pipeline.
|
|
|
|
|
|
|
|
|
|
Properties:
|
|
|
|
|
fitness (float): The fitness of the model, which is equal to top-5 accuracy.
|
|
|
|
|
results_dict (Dict[str, Union[float, str]]): A dictionary containing the classification metrics and fitness.
|
|
|
|
|
keys (List[str]): A list of keys for the results_dict.
|
|
|
|
|
|
|
|
|
|
Methods:
|
|
|
|
|
process(targets, pred): Processes the targets and predictions to compute classification metrics.
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
def __init__(self) -> None:
|
|
|
|
|
self.top1 = 0
|
|
|
|
|
self.top5 = 0
|
|
|
|
|
self.speed = {'preprocess': 0.0, 'inference': 0.0, 'loss': 0.0, 'postprocess': 0.0}
|
|
|
|
|
|
|
|
|
|
def __getattr__(self, attr):
|
|
|
|
|
name = self.__class__.__name__
|
|
|
|
|
raise AttributeError(f"'{name}' object has no attribute '{attr}'. See valid attributes below.\n{self.__doc__}")
|
|
|
|
|
|
|
|
|
|
def process(self, targets, pred):
|
|
|
|
|
# target classes and predicted classes
|
|
|
|
|
pred, targets = torch.cat(pred), torch.cat(targets)
|
|
|
|
|
|
Glenn Jocher
2 years ago
committed by
GitHub