diff --git a/pyproject.toml b/pyproject.toml index 7414ede0..fe7f10e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -235,13 +235,6 @@ module = [ "supervision.key_points.annotators", "supervision.key_points.core", "supervision.key_points.skeletons", - "supervision.metrics.core", - "supervision.metrics.detection", - "supervision.metrics.f1_score", - "supervision.metrics.mean_average_precision", - "supervision.metrics.mean_average_recall", - "supervision.metrics.precision", - "supervision.metrics.recall", "supervision.metrics.utils.utils", "supervision.tracker.byte_tracker.core", "supervision.tracker.byte_tracker.kalman_filter", diff --git a/supervision/metrics/core.py b/supervision/metrics/core.py index 75bc5152..ad6a1781 100644 --- a/supervision/metrics/core.py +++ b/supervision/metrics/core.py @@ -11,7 +11,7 @@ class Metric(ABC): """ @abstractmethod - def update(self, *args, **kwargs) -> Metric: + def update(self, *args: Any, **kwargs: Any) -> Metric: """ Add data to the metric, without computing the result. Return the metric itself to allow method chaining. @@ -26,7 +26,7 @@ class Metric(ABC): raise NotImplementedError @abstractmethod - def compute(self, *args, **kwargs) -> Any: + def compute(self, *args: Any, **kwargs: Any) -> Any: """ Compute the metric from the internal state and return the result. """ diff --git a/supervision/metrics/detection.py b/supervision/metrics/detection.py index 67c3a798..42404145 100644 --- a/supervision/metrics/detection.py +++ b/supervision/metrics/detection.py @@ -18,12 +18,13 @@ def detections_to_tensor( ) -> np.ndarray: """ Convert Supervision Detections to numpy tensors for further computation + Args: - detections (sv.Detections): Detections/Targets in the format of sv.Detections - with_confidence (bool): Whether to include confidence in the tensor + detections: Detections/Targets in the format of sv.Detections + with_confidence: Whether to include confidence in the tensor + Returns: - (np.ndarray): Detections as numpy tensors as in (xyxy, class_id, - confidence) order + Detections as numpy tensors as in (xyxy, class_id, confidence) order """ if detections.class_id is None: raise ValueError( @@ -39,10 +40,13 @@ def detections_to_tensor( ) arrays_to_concat.append(np.expand_dims(detections.confidence, 1)) - return np.concatenate(arrays_to_concat, axis=1) + result: np.ndarray = np.concatenate(arrays_to_concat, axis=1) + return result -def validate_input_tensors(predictions: list[np.ndarray], targets: list[np.ndarray]): +def validate_input_tensors( + predictions: list[np.ndarray], targets: list[np.ndarray] +) -> None: """ Checks for shape consistency of input tensors. """ @@ -76,13 +80,12 @@ class ConfusionMatrix: Confusion matrix for object detection tasks. Attributes: - matrix (np.ndarray): An 2D `np.ndarray` of shape - `(len(classes) + 1, len(classes) + 1)` + matrix: An 2D `np.ndarray` of shape `(len(classes) + 1, len(classes) + 1)` containing the number of `TP`, `FP`, `FN` and `TN` for each class. - classes (List[str]): Model class names. - conf_threshold (float): Detection confidence threshold between `0` and `1`. + classes: Model class names. + conf_threshold: Detection confidence threshold between `0` and `1`. Detections with lower confidence will be excluded from the matrix. - iou_threshold (float): Detection IoU threshold between `0` and `1`. + iou_threshold: Detection IoU threshold between `0` and `1`. Detections with lower IoU will be classified as `FP`. """ @@ -104,16 +107,16 @@ class ConfusionMatrix: Calculate confusion matrix based on predicted and ground-truth detections. Args: - targets (List[Detections]): Detections objects from ground-truth. - predictions (List[Detections]): Detections objects predicted by the model. - classes (List[str]): Model class names. - conf_threshold (float): Detection confidence threshold between `0` and `1`. + targets: Detections objects from ground-truth. + predictions: Detections objects predicted by the model. + classes: Model class names. + conf_threshold: Detection confidence threshold between `0` and `1`. Detections with lower confidence will be excluded. - iou_threshold (float): Detection IoU threshold between `0` and `1`. + iou_threshold: Detection IoU threshold between `0` and `1`. Detections with lower IoU will be classified as `FP`. Returns: - ConfusionMatrix: New instance of ConfusionMatrix. + New instance of ConfusionMatrix. Examples: >>> import numpy as np @@ -169,22 +172,22 @@ class ConfusionMatrix: Calculate confusion matrix based on predicted and ground-truth detections. Args: - predictions (List[np.ndarray]): Each element of the list describes a single + predictions: Each element of the list describes a single image and has `shape = (M, 6)` where `M` is the number of detected objects. Each row is expected to be in `(x_min, y_min, x_max, y_max, class, conf)` format. - targets (List[np.ndarray]): Each element of the list describes a single + targets: Each element of the list describes a single image and has `shape = (N, 5)` where `N` is the number of ground-truth objects. Each row is expected to be in `(x_min, y_min, x_max, y_max, class)` format. - classes (List[str]): Model class names. - conf_threshold (float): Detection confidence threshold between `0` and `1`. + classes: Model class names. + conf_threshold: Detection confidence threshold between `0` and `1`. Detections with lower confidence will be excluded. - iou_threshold (float): Detection iou threshold between `0` and `1`. + iou_threshold: Detection iou threshold between `0` and `1`. Detections with lower iou will be classified as `FP`. Returns: - ConfusionMatrix: New instance of ConfusionMatrix. + New instance of ConfusionMatrix. Examples: >>> import supervision as sv @@ -244,22 +247,22 @@ class ConfusionMatrix: Calculate confusion matrix for a batch of detections for a single image. Args: - predictions (np.ndarray): Batch prediction. Describes a single image and + predictions: Batch prediction. Describes a single image and has `shape = (M, 6)` where `M` is the number of detected objects. Each row is expected to be in `(x_min, y_min, x_max, y_max, class, conf)` format. - targets (np.ndarray): Batch target labels. Describes a single image and + targets: Batch target labels. Describes a single image and has `shape = (N, 5)` where `N` is the number of ground-truth objects. Each row is expected to be in `(x_min, y_min, x_max, y_max, class)` format. - num_classes (int): Number of classes. - conf_threshold (float): Detection confidence threshold between `0` and `1`. + num_classes: Number of classes. + conf_threshold: Detection confidence threshold between `0` and `1`. Detections with lower confidence will be excluded. - iou_threshold (float): Detection iou threshold between `0` and `1`. + iou_threshold: Detection iou threshold between `0` and `1`. Detections with lower iou will be classified as `FP`. Returns: - np.ndarray: Confusion matrix based on a single image. + Confusion matrix based on a single image. """ result_matrix = np.zeros((num_classes + 1, num_classes + 1)) @@ -304,8 +307,8 @@ class ConfusionMatrix: for i, detection_class_value in enumerate(detection_classes): if not any(matched_detection_idx == i): result_matrix[num_classes, detection_class_value] += 1 # FP - - return result_matrix + final_result_matrix: np.ndarray = result_matrix + return final_result_matrix @staticmethod def _drop_extra_matches(matches: np.ndarray) -> np.ndarray: @@ -332,16 +335,16 @@ class ConfusionMatrix: Calculate confusion matrix from dataset and callback function. Args: - dataset (DetectionDataset): Object detection dataset used for evaluation. - callback (Callable[[np.ndarray], Detections]): Function that takes an image - as input and returns Detections object. - conf_threshold (float): Detection confidence threshold between `0` and `1`. + dataset: Object detection dataset used for evaluation. + callback: Function that takes an image as input and returns a + Detections object. + conf_threshold: Detection confidence threshold between `0` and `1`. Detections with lower confidence will be excluded. - iou_threshold (float): Detection IoU threshold between `0` and `1`. + iou_threshold: Detection IoU threshold between `0` and `1`. Detections with lower IoU will be classified as `FP`. Returns: - ConfusionMatrix: New instance of ConfusionMatrix. + New instance of ConfusionMatrix. Example: ```python @@ -394,16 +397,16 @@ class ConfusionMatrix: Create confusion matrix plot and save it at selected location. Args: - save_path (Optional[str]): Path to save the plot. If not provided, + save_path: Path to save the plot. If not provided, plot will be displayed. - title (Optional[str]): Title of the plot. - classes (Optional[List[str]]): List of classes to be displayed on the plot. + title: Title of the plot. + classes: List of classes to be displayed on the plot. If not provided, all classes will be displayed. - normalize (bool): If True, normalize the confusion matrix. - fig_size (Tuple[int, int]): Size of the plot. + normalize: If True, normalize the confusion matrix. + fig_size: Size of the plot. Returns: - matplotlib.figure.Figure: Confusion matrix plot. + Confusion matrix plot. """ array = self.matrix.copy() @@ -493,13 +496,13 @@ class MeanAveragePrecision: Mean Average Precision for object detection tasks. Attributes: - map50_95 (float): Mean Average Precision (mAP) calculated over IoU thresholds + map50_95: Mean Average Precision (mAP) calculated over IoU thresholds ranging from `0.50` to `0.95` with a step size of `0.05`. - map50 (float): Mean Average Precision (mAP) calculated specifically at + map50: Mean Average Precision (mAP) calculated specifically at an IoU threshold of `0.50`. - map75 (float): Mean Average Precision (mAP) calculated specifically at + map75: Mean Average Precision (mAP) calculated specifically at an IoU threshold of `0.75`. - per_class_ap50_95 (np.ndarray): Average Precision (AP) values calculated over + per_class_ap50_95: Average Precision (AP) values calculated over IoU thresholds ranging from `0.50` to `0.95` with a step size of `0.05`, provided for each individual class. """ @@ -519,10 +522,10 @@ class MeanAveragePrecision: Calculate mean average precision based on predicted and ground-truth detections. Args: - targets (List[Detections]): Detections objects from ground-truth. - predictions (List[Detections]): Detections objects predicted by the model. + targets: Detections objects from ground-truth. + predictions: Detections objects predicted by the model. Returns: - MeanAveragePrecision: New instance of ConfusionMatrix. + New instance of ConfusionMatrix. Examples: >>> import numpy as np @@ -569,11 +572,11 @@ class MeanAveragePrecision: Calculate mean average precision from dataset and callback function. Args: - dataset (DetectionDataset): Object detection dataset used for evaluation. - callback (Callable[[np.ndarray], Detections]): Function that takes + dataset: Object detection dataset used for evaluation. + callback: Function that takes an image as input and returns Detections object. Returns: - MeanAveragePrecision: New instance of MeanAveragePrecision. + New instance of MeanAveragePrecision. Example: ```python @@ -617,16 +620,16 @@ class MeanAveragePrecision: detections at different threshold. Args: - predictions (List[np.ndarray]): Each element of the list describes + predictions: Each element of the list describes a single image and has `shape = (M, 6)` where `M` is the number of detected objects. Each row is expected to be in `(x_min, y_min, x_max, y_max, class, conf)` format. - targets (List[np.ndarray]): Each element of the list describes a single + targets: Each element of the list describes a single image and has `shape = (N, 5)` where `N` is the number of ground-truth objects. Each row is expected to be in `(x_min, y_min, x_max, y_max, class)` format. Returns: - MeanAveragePrecision: New instance of MeanAveragePrecision. + New instance of MeanAveragePrecision. Examples: >>> import supervision as sv @@ -691,7 +694,7 @@ class MeanAveragePrecision: map50_95 = average_precisions.mean() else: map50, map75, map50_95 = 0, 0, 0 - average_precisions = [] + average_precisions = np.array([]) return cls( map50_95=map50_95, @@ -707,11 +710,11 @@ class MeanAveragePrecision: the recall and precision curves. Args: - recall (np.ndarray): The recall curve. - precision (np.ndarray): The precision curve. + recall: The recall curve. + precision: The precision curve. Returns: - float: Average precision. + Average precision. """ extended_recall = np.concatenate(([0.0], recall, [1.0])) extended_precision = np.concatenate(([1.0], precision, [0.0])) @@ -722,8 +725,18 @@ class MeanAveragePrecision: interpolated_precision = np.interp( interpolated_recall_levels, extended_recall, max_accumulated_precision ) - average_precision = np.trapz(interpolated_precision, interpolated_recall_levels) - return average_precision + + # Check if we are running on NumPy 2.0+ or older + if hasattr(np, "trapezoid"): + average_precision = np.trapezoid( + interpolated_precision, interpolated_recall_levels + ) + else: + average_precision = np.trapz( # type: ignore[attr-defined] + interpolated_precision, interpolated_recall_levels + ) + + return float(average_precision) @staticmethod def _match_detection_batch( @@ -733,18 +746,18 @@ class MeanAveragePrecision: Match predictions with target labels based on IoU levels. Args: - predictions (np.ndarray): Batch prediction. Describes a single image and + predictions: Batch prediction. Describes a single image and has `shape = (M, 6)` where `M` is the number of detected objects. Each row is expected to be in `(x_min, y_min, x_max, y_max, class, conf)` format. - targets (np.ndarray): Batch target labels. Describes a single image and + targets: Batch target labels. Describes a single image and has `shape = (N, 5)` where `N` is the number of ground-truth objects. Each row is expected to be in `(x_min, y_min, x_max, y_max, class)` format. - iou_thresholds (np.ndarray): Array contains different IoU thresholds. + iou_thresholds: Array contains different IoU thresholds. Returns: - np.ndarray: Matched prediction with target labels result. + Matched prediction with target labels result. """ num_predictions, num_iou_levels = predictions.shape[0], iou_thresholds.shape[0] correct = np.zeros((num_predictions, num_iou_levels), dtype=bool) @@ -765,8 +778,8 @@ class MeanAveragePrecision: matches = matches[np.unique(matches[:, 0], return_index=True)[1]] correct[matches[:, 1].astype(int), i] = True - - return correct + result: np.ndarray = correct + return result @staticmethod def _average_precisions_per_class( @@ -781,14 +794,14 @@ class MeanAveragePrecision: Source: https://github.com/rafaelpadilla/Object-Detection-Metrics. Args: - matches (np.ndarray): True positives. - prediction_confidence (np.ndarray): Objectness value from 0-1. - prediction_class_ids (np.ndarray): Predicted object classes. - true_class_ids (np.ndarray): True object classes. - eps (float): Small value to prevent division by zero. + matches: True positives. + prediction_confidence: Objectness value from 0-1. + prediction_class_ids: Predicted object classes. + true_class_ids: True object classes. + eps: Small value to prevent division by zero. Returns: - np.ndarray: Average precision for different IoU levels. + Average precision for different IoU levels. """ sorted_indices = np.argsort(-prediction_confidence) matches = matches[sorted_indices] @@ -819,4 +832,5 @@ class MeanAveragePrecision: ) ) - return average_precisions + result: np.ndarray = average_precisions + return result diff --git a/supervision/metrics/f1_score.py b/supervision/metrics/f1_score.py index 4616014f..3c1c710b 100644 --- a/supervision/metrics/f1_score.py +++ b/supervision/metrics/f1_score.py @@ -2,7 +2,7 @@ from __future__ import annotations from copy import deepcopy from dataclasses import dataclass -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import numpy as np from matplotlib import pyplot as plt @@ -68,8 +68,8 @@ class F1Score(Metric): Initialize the F1Score metric. Args: - metric_target (MetricTarget): The type of detection data to use. - averaging_method (AveragingMethod): The averaging method used to compute the + metric_target: The type of detection data to use. + averaging_method: The averaging method used to compute the F1 scores. Determines how the F1 scores are aggregated across classes. """ self._metric_target = metric_target @@ -94,11 +94,11 @@ class F1Score(Metric): Add new predictions and targets to the metric, but do not compute the result. Args: - predictions (Union[Detections, List[Detections]]): The predicted detections. - targets (Union[Detections, List[Detections]]): The target detections. + predictions: The predicted detections. + targets: The target detections. Returns: - (F1Score): The updated metric instance. + The updated metric instance. """ if not isinstance(predictions, list): predictions = [predictions] @@ -122,7 +122,7 @@ class F1Score(Metric): data, at different IoU thresholds. Returns: - (F1ScoreResult): The F1 score metric result. + The F1 score metric result. """ result = self._compute(self._predictions_list, self._targets_list) @@ -149,7 +149,7 @@ class F1Score(Metric): self, predictions_list: list[Detections], targets_list: list[Detections] ) -> F1ScoreResult: iou_thresholds = np.linspace(0.5, 0.95, 10) - stats = [] + stats: list[Any] = [] for predictions, targets in zip(predictions_list, targets_list): prediction_contents = self._detections_content(predictions) @@ -181,7 +181,14 @@ class F1Score(Metric): ) matches = self._match_detection_batch( - predictions.class_id, targets.class_id, iou, iou_thresholds + predictions.class_id + if predictions.class_id is not None + else np.array([]), + targets.class_id + if targets.class_id is not None + else np.array([]), + iou, + iou_thresholds, ) stats.append( ( @@ -283,7 +290,8 @@ class F1Score(Metric): correct[matches[:, 1].astype(int), i] = True - return correct + result_correct: np.ndarray = correct + return result_correct @staticmethod def _compute_confusion_matrix( @@ -298,18 +306,18 @@ class F1Score(Metric): Assumes the matches and prediction_class_ids are sorted by confidence in descending order. - Arguments: - sorted_matches: np.ndarray, bool, shape (P, Th), that is True + Args: + sorted_matches: shape (P, Th), that is True if the prediction is a true positive at the given IoU threshold. - sorted_prediction_class_ids: np.ndarray, int, shape (P,), containing + sorted_prediction_class_ids: shape (P,), containing the class id for each prediction. - unique_classes: np.ndarray, int, shape (C,), containing the unique + unique_classes: shape (C,), containing the unique class ids. - class_counts: np.ndarray, int, shape (C,), containing the number + class_counts: shape (C,), containing the number of true instances for each class. Returns: - np.ndarray, shape (C, Th, 3), containing the true positives, false + shape (C, Th, 3), containing the true positives, false positives, and false negatives for each class and IoU threshold. """ @@ -338,7 +346,8 @@ class F1Score(Metric): [true_positives, false_positives, false_negatives], axis=1 ) - return confusion_matrix + result_confusion_matrix: np.ndarray = confusion_matrix + return result_confusion_matrix @staticmethod def _compute_f1(confusion_matrix: np.ndarray) -> np.ndarray: @@ -346,11 +355,11 @@ class F1Score(Metric): Broadcastable function, computing the F1 score from the confusion matrix. Arguments: - confusion_matrix: np.ndarray, shape (N, ..., 3), where the last dimension + confusion_matrix: shape (N, ..., 3), where the last dimension contains the true positives, false positives, and false negatives. Returns: - np.ndarray, shape (N, ...), containing the F1 score for each element. + shape (N, ...), containing the F1 score for each element. """ if not confusion_matrix.shape[-1] == 3: raise ValueError( @@ -365,7 +374,8 @@ class F1Score(Metric): denominator = 2 * true_positives + false_positives + false_negatives f1_score = np.where(denominator == 0, 0, 2 * true_positives / denominator) - return f1_score + result_f1_score: np.ndarray = f1_score + return result_f1_score def _detections_content(self, detections: Detections) -> np.ndarray: """Return boxes, masks or oriented bounding boxes from detections.""" @@ -380,17 +390,21 @@ class F1Score(Metric): if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: obb = detections.data.get(ORIENTED_BOX_COORDINATES) if obb is not None and len(obb) > 0: - return np.array(obb, dtype=np.float32) + result_obb: np.ndarray = np.array(obb, dtype=np.float32) + return result_obb return self._make_empty_content() raise ValueError(f"Invalid metric target: {self._metric_target}") def _make_empty_content(self) -> np.ndarray: if self._metric_target == MetricTarget.BOXES: - return np.empty((0, 4), dtype=np.float32) + empty_boxes: np.ndarray = np.empty((0, 4), dtype=np.float32) + return empty_boxes if self._metric_target == MetricTarget.MASKS: - return np.empty((0, 0, 0), dtype=bool) + empty_masks: np.ndarray = np.empty((0, 0, 0), dtype=bool) + return empty_masks if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: - return np.empty((0, 4, 2), dtype=np.float32) + empty_obb: np.ndarray = np.empty((0, 4, 2), dtype=np.float32) + return empty_obb raise ValueError(f"Invalid metric target: {self._metric_target}") def _filter_detections_by_size( @@ -448,24 +462,24 @@ class F1ScoreResult: Defaults to `0` if no detections or targets were provided. Attributes: - metric_target (MetricTarget): the type of data used for the metric - + metric_target: the type of data used for the metric - boxes, masks or oriented bounding boxes. - averaging_method (AveragingMethod): the averaging method used to compute the + averaging_method: the averaging method used to compute the F1 scores. Determines how the F1 scores are aggregated across classes. - f1_50 (float): the F1 score at IoU threshold of `0.5`. - f1_75 (float): the F1 score at IoU threshold of `0.75`. - f1_scores (np.ndarray): the F1 scores at each IoU threshold. + f1_50: the F1 score at IoU threshold of `0.5`. + f1_75: the F1 score at IoU threshold of `0.75`. + f1_scores: the F1 scores at each IoU threshold. Shape: `(num_iou_thresholds,)` - f1_per_class (np.ndarray): the F1 scores per class and IoU threshold. + f1_per_class: the F1 scores per class and IoU threshold. Shape: `(num_target_classes, num_iou_thresholds)` - iou_thresholds (np.ndarray): the IoU thresholds used in the calculations. - matched_classes (np.ndarray): the class IDs of all matched classes. + iou_thresholds: the IoU thresholds used in the calculations. + matched_classes: the class IDs of all matched classes. Corresponds to the rows of `f1_per_class`. - small_objects (Optional[F1ScoreResult]): the F1 metric results + small_objects: the F1 metric results for small objects (area < 32²). - medium_objects (Optional[F1ScoreResult]): the F1 metric results + medium_objects: the F1 metric results for medium objects (32² ≤ area < 96²). - large_objects (Optional[F1ScoreResult]): the F1 metric results + large_objects: the F1 metric results for large objects (area ≥ 96²). """ @@ -474,11 +488,11 @@ class F1ScoreResult: @property def f1_50(self) -> float: - return self.f1_scores[0] + return float(self.f1_scores[0]) @property def f1_75(self) -> float: - return self.f1_scores[5] + return float(self.f1_scores[5]) f1_scores: np.ndarray f1_per_class: np.ndarray @@ -544,7 +558,7 @@ class F1ScoreResult: Convert the result to a pandas DataFrame. Returns: - (pd.DataFrame): The result as a DataFrame. + The result as a DataFrame. """ ensure_pandas_installed() import pandas as pd @@ -569,7 +583,7 @@ class F1ScoreResult: return pd.DataFrame(pandas_data, index=[0]) - def plot(self): + def plot(self) -> None: """ Plot the F1 results. diff --git a/supervision/metrics/mean_average_precision.py b/supervision/metrics/mean_average_precision.py index 6354f966..2be64f0a 100644 --- a/supervision/metrics/mean_average_precision.py +++ b/supervision/metrics/mean_average_precision.py @@ -12,8 +12,8 @@ from typing import TYPE_CHECKING, Any import numpy as np from matplotlib import pyplot as plt -from supervision import box_iou_batch_with_jaccard from supervision.detection.core import Detections +from supervision.detection.utils.iou_and_nms import box_iou_batch_with_jaccard from supervision.draw.color import LEGACY_COLOR_PALETTE from supervision.metrics.core import Metric, MetricTarget from supervision.metrics.utils.utils import ensure_pandas_installed @@ -30,25 +30,22 @@ class MeanAveragePrecisionResult: Defaults to `0` when no detections or targets are present. Attributes: - metric_target (MetricTarget): the type of data used for the metric - + metric_target: the type of data used for the metric - boxes, masks or oriented bounding boxes. - class_agnostic (bool): When computing class-agnostic results, class ID + is_class_agnostic: When computing class-agnostic results, class ID is set to `-1`. - mAP_map50_95 (float): the mAP score at IoU thresholds from `0.5` to `0.95`. - mAP_map50 (float): the mAP score at IoU threshold of `0.5`. - mAP_map75 (float): the mAP score at IoU threshold of `0.75`. - mAP_scores (np.ndarray): the mAP scores at each IoU threshold. + mAP_scores: the mAP scores at each IoU threshold. Shape: `(num_iou_thresholds,)` - ap_per_class (np.ndarray): the average precision scores per + ap_per_class: the average precision scores per class and IoU threshold. Shape: `(num_target_classes, num_iou_thresholds)` - iou_thresholds (np.ndarray): the IoU thresholds used in the calculations. - matched_classes (np.ndarray): the class IDs of all matched classes. + iou_thresholds: the IoU thresholds used in the calculations. + matched_classes: the class IDs of all matched classes. Corresponds to the rows of `ap_per_class`. - small_objects (Optional[MeanAveragePrecisionResult]): the mAP results + small_objects: the mAP results for small objects (area < 32²). - medium_objects (Optional[MeanAveragePrecisionResult]): the mAP results + medium_objects: the mAP results for medium objects (32² ≤ area < 96²). - large_objects (Optional[MeanAveragePrecisionResult]): the mAP results + large_objects: the mAP results for large objects (area ≥ 96²). """ @@ -57,19 +54,22 @@ class MeanAveragePrecisionResult: @property def map50_95(self) -> float: + """the mAP score at IoU thresholds from `0.5` to `0.95`.""" valid_scores = self.mAP_scores[self.mAP_scores > -1] if len(valid_scores) > 0: - return valid_scores.mean() + return float(valid_scores.mean()) else: return -1 @property def map50(self) -> float: - return self.mAP_scores[0] + """the mAP score at IoU threshold of `0.5`.""" + return float(self.mAP_scores[0]) @property def map75(self) -> float: - return self.mAP_scores[5] + """the mAP score at IoU threshold of `0.75`.""" + return float(self.mAP_scores[5]) mAP_scores: np.ndarray ap_per_class: np.ndarray @@ -95,6 +95,20 @@ class MeanAveragePrecisionResult: Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.629 ``` """ + if ( + self.small_objects is None + or self.medium_objects is None + or self.large_objects is None + ): + return ( + f"Average Precision (AP) @[ IoU=0.50:0.95 | area= all | " + f"maxDets=100 ] = {self.map50_95:.3f}\n" + f"Average Precision (AP) @[ IoU=0.50 | area= all | " + f"maxDets=100 ] = {self.map50:.3f}\n" + f"Average Precision (AP) @[ IoU=0.75 | area= all | " + f"maxDets=100 ] = {self.map75:.3f}" + ) + return ( f"Average Precision (AP) @[ IoU=0.50:0.95 | area= all | " f"maxDets=100 ] = {self.map50_95:.3f}\n" @@ -115,7 +129,7 @@ class MeanAveragePrecisionResult: Convert the result to a pandas DataFrame. Returns: - (pd.DataFrame): The result as a DataFrame. + The result as a DataFrame. """ ensure_pandas_installed() import pandas as pd @@ -145,7 +159,7 @@ class MeanAveragePrecisionResult: index=[0], ) - def plot(self): + def plot(self) -> None: """ Plot the mAP results. @@ -224,13 +238,19 @@ class EvaluationDataset: """ Constructor of EvaluationDataset object used to evaluate models with Mean Average Precision. + Args: - targets (dict): The targets (ground truth) of the dataset in a the - COCO format. + targets: The targets (ground truth) of the dataset in a the + COCO format. """ # Initialize members - self.dataset, self.anns, self.cats, self.imgs = dict(), dict(), dict(), dict() - self.img_to_anns, self.cat_to_imgs = defaultdict(list), defaultdict(list) + # Initialize members + self.dataset: dict[str, Any] = dict() + self.anns: dict[int, Any] = dict() + self.cats: dict[int, Any] = dict() + self.imgs: dict[int, Any] = dict() + self.img_to_anns: dict[int, list[Any]] = defaultdict(list) + self.cat_to_imgs: dict[int, list[int]] = defaultdict(list) if targets is None: return @@ -240,10 +260,10 @@ class EvaluationDataset: self.create_class_members() @classmethod - def empty(cls): + def empty(cls) -> EvaluationDataset: return cls(targets=None) - def create_class_members(self): + def create_class_members(self) -> None: """ Create index elements for the dataset. """ @@ -275,25 +295,26 @@ class EvaluationDataset: def get_annotation_ids( self, - img_ids: list[int] = [], - cat_ids: list[int] = [], - area_range: tuple[float, float] = [], + img_ids: list[int] | None = None, + cat_ids: list[int] | None = None, + area_range: tuple[float, float] | None = None, iscrowd: bool = False, - ): + ) -> list[int]: """ Get annotation ids that satisfy given filter conditions. + Args: - img_ids (list): ids of the images that we want to retrieve. - cat_ids (list): ids of the categories that we want to retrieve. - area_range (tuple): area range of the annotations that we want to retrieve - in the format [min_area, max_area]. - iscrowd (bool): if annotations to retrieve are `iscrowded=1`. + img_ids: ids of the images that we want to retrieve. + cat_ids: ids of the categories that we want to retrieve. + area_range: area range of the annotations that we want to retrieve + in the format [min_area, max_area]. + iscrowd: if annotations to retrieve are `iscrowded=1`. """ # If there are no filters, we use all annotations - if len(img_ids) == len(cat_ids) == len(area_range) == 0: + if not img_ids and not cat_ids and not area_range: anns = self.dataset["annotations"] else: - if len(img_ids) != 0: + if img_ids: lists = [ self.img_to_anns[img_id] for img_id in img_ids @@ -306,18 +327,16 @@ class EvaluationDataset: # Filter by category anns = ( anns - if len(cat_ids) == 0 + if not cat_ids else [ann for ann in anns if ann["category_id"] in cat_ids] ) # Filter by area anns = ( anns - if len(area_range) == 0 + if not area_range else [ - ann - for ann in anns - if ann["area"] > area_range[0] and ann["area"] < area_range[1] + ann for ann in anns if area_range[0] < ann["area"] < area_range[1] ] ) @@ -330,21 +349,23 @@ class EvaluationDataset: def get_category_ids( self, - cat_names: list[str] = [], - supercategory_names: list[str] = [], - cat_ids: list[int] = [], + cat_names: list[str] | None = None, + supercategory_names: list[str] | None = None, + cat_ids: list[int] | None = None, ) -> list[int]: """ Get category ids that satisfy given filter conditions. + Args: - cat_names (list): names of the categories to retrieve. - supercategory_names (list): names of the supercategories to retrieve. - cat_ids (list): ids of the categories to retrieve. + cat_names: names of the categories to retrieve. + supercategory_names: names of the supercategories to retrieve. + cat_ids: ids of the categories to retrieve. + Returns: - ids (list): integer array of category ids. + ids: integer array of category ids. """ # If there are no filters, we use all categories - if len(cat_names) == len(supercategory_names) == len(cat_ids) == 0: + if not cat_names and not supercategory_names and not cat_ids: cats = self.dataset["categories"] else: cats = self.dataset["categories"] @@ -352,14 +373,14 @@ class EvaluationDataset: # Filter by name cats = ( cats - if len(cat_names) == 0 + if not cat_names else [cat for cat in cats if cat["name"] in cat_names] ) # Filter by supercategory cats = ( cats - if len(supercategory_names) == 0 + if not supercategory_names else [ cat for cat in cats if cat["supercategory"] in supercategory_names ] @@ -367,54 +388,63 @@ class EvaluationDataset: # Filter by id cats = ( - cats - if len(cat_ids) == 0 - else [cat for cat in cats if cat["id"] in cat_ids] + cats if not cat_ids else [cat for cat in cats if cat["id"] in cat_ids] ) ids = [cat["id"] for cat in cats] return ids def get_image_ids( self, - img_ids: list[int] = [], - cat_ids: list[int] = [], + img_ids: list[int] | None = None, + cat_ids: list[int] | None = None, ) -> list[int]: """ Get image ids that satisfy given filter conditions. + Args: - img_ids (list): ids of the images to retrieve. - cat_ids (list): ids of the categories to retrieve. + img_ids: ids of the images to retrieve. + cat_ids: ids of the categories to retrieve. + Returns: - ids (list): integer array of image ids. + ids: integer array of image ids. """ # If there are no filters, we use all images - if len(img_ids) == len(cat_ids) == 0: + if not img_ids and not cat_ids: ids = self.imgs.keys() return list(ids) - ids = set(img_ids) - for i, cat_id in enumerate(cat_ids): - if i == 0 and len(ids) == 0: - ids = set(self.cat_to_imgs[cat_id]) - else: - ids &= set(self.cat_to_imgs[cat_id]) - return list(ids) + ids_set = set(img_ids) if img_ids else set() - def get_annotations(self, ids: list[int] = []) -> list[dict]: + if cat_ids: + for i, cat_id in enumerate(cat_ids): + if i == 0 and not ids_set: + ids_set = set(self.cat_to_imgs[cat_id]) + else: + ids_set &= set(self.cat_to_imgs[cat_id]) + + return list(ids_set) + + def get_annotations(self, ids: list[int] | None = None) -> list[dict[str, Any]]: """ Get annotations with the specified ids. + Args: - ids (list): integer ids specifying annotations. + ids: integer ids specifying annotations. + Returns: - anns (list): loaded annotations. + anns: loaded annotations. """ + if ids is None: + return [] return [self.anns[idx] for idx in ids] - def load_predictions(self, predictions: list[dict]) -> EvaluationDataset: + def load_predictions(self, predictions: list[dict[str, Any]]) -> EvaluationDataset: """ Load prediction result into an EvaluationDataset object. + Args: - predictions (list): prediction result. + predictions: prediction result. + Returns: EvaluationDataset object representing the predictions. """ @@ -464,7 +494,9 @@ class EvaluationDataset: # Make segmentation from bounding box coordinates if "segmentation" not in pred: pred["segmentation"] = [[x1, y1, x1, y2, x2, y2, x2, y1]] - pred["area"] = w * h + # Use provided area if available + if "area" not in pred: + pred["area"] = w * h pred["id"] = idx + 1 # For predictions we set iscrowd to 0 pred["iscrowd"] = 0 @@ -498,10 +530,12 @@ class COCOEvaluatorParameters: Parameters for COCOEvaluator """ - def __init__(self): + def __init__(self) -> None: """Initialize all parameters for evaluation""" - self.img_ids, self.cat_ids = [], [] + self.img_ids: list[int] = [] + self.cat_ids: list[int] = [] + # IoU thresholds [0.5, 0.55, 0.6, 0.65, ..., 0.95] self.iou_thrs = np.linspace( 0.5, 0.95, int(np.round((0.95 - 0.5) / 0.05)) + 1, endpoint=True @@ -513,7 +547,7 @@ class COCOEvaluatorParameters: # 3 maximum detection thresholds [1, 10, 100] self.max_dets = [1, 10, 100] # Area ranges [0, 1e5], [0, 32], [32, 96], [96, 1e5] - self.area_range = [ + self.area_range: list[list[float]] = [ [0, MAX_ALL_OBJECT_AREA], [0, SMALL_OBJECT_AREA], [SMALL_OBJECT_AREA, MEDIUM_OBJECT_AREA], @@ -533,8 +567,8 @@ class COCOEvaluator: Constructor of COCOEvaluator object. Args: - coco_targets (EvaluationDataset): The dataset with the ground truths. - coco_predictions (EvaluationDataset): The dataset with the predictions. + coco_targets: The dataset with the ground truths. + coco_predictions: The dataset with the predictions. """ if coco_targets is None: raise ValueError("coco_targets must be provided") @@ -546,23 +580,23 @@ class COCOEvaluator: # List of dictionaries containing the evaluation results # len(eval_imgs) = (categories) * (area_ranges) * (images) # For COCO 2017: len(eval_images) = 80 * 4 * 5000 = 1600000 - self.eval_imgs = defaultdict(list) + self.eval_imgs: Any = defaultdict(list) # Dictionary of accumulated results - self.results = {} + self.results: dict[str, Any] = {} # Dictionary of targets for evaluation - self._targets = defaultdict(list) - self._predictions = defaultdict(list) + self._targets: defaultdict[tuple[int, int], list[Any]] = defaultdict(list) + self._predictions: defaultdict[tuple[int, int], list[Any]] = defaultdict(list) # Parameters for evaluation self.params = COCOEvaluatorParameters() # List of results summarization - self.stats = [] + self.stats: list[Any] = [] # Dictionary of IOUs between all targets and predictions - self.ious = {} + self.ious: dict[tuple[int, int], Any] = {} # Set image and category ids self.params.img_ids = sorted(self.coco_targets.get_image_ids()) self.params.cat_ids = sorted(self.coco_targets.get_category_ids()) - def _prepare_targets_and_predictions(self): + def _prepare_targets_and_predictions(self) -> None: """ Prepare targets and predictions for evaluation. """ @@ -602,11 +636,11 @@ class COCOEvaluator: category. Args: - img_id (int): The image id. - cat_id (int): The category id. + img_id: The image id. + cat_id: The category id. Returns: - np.ndarray: The IoU between the targets and predictions. + The IoU between the targets and predictions. """ gt = self._targets[img_id, cat_id] @@ -614,7 +648,8 @@ class COCOEvaluator: # If there is nothing to evaluate if len(gt) == 0 and len(dt) == 0: - return np.array([]) + empty_result: np.ndarray = np.array([], dtype=np.float64) + return empty_result # Sort predictions by highest score first inds = np.argsort([-d["score"] for d in dt], kind="stable") @@ -629,28 +664,32 @@ class COCOEvaluator: dt_boxes = [d["bbox"] for d in dt] # Get the iscrowd flag for each gt - is_crowd = [int(o["iscrowd"]) for o in gt] + is_crowd = [bool(o["iscrowd"]) for o in gt] # Compute iou between each prediction a and gt region iou = box_iou_batch_with_jaccard(gt_boxes, dt_boxes, is_crowd) return iou def _evaluate_image( - self, img_id: int, cat_id: int, area_range: tuple[int, int], max_det: int + self, + img_id: int, + cat_id: int, + area_range: list[float] | tuple[float, float], + max_det: int, ) -> dict[str, Any] | None: """ Perform evaluation for single category and image. Args: - img_id (int): The image id. - cat_id (int): The category id. - area_range (Tuple[int, int]): The area range. - max_det (int): The maximum number of detections. + img_id: The image id. + cat_id: The category id. + area_range: The area range. + max_det: The maximum number of detections. Returns: - Dict[str, Any]: The evaluation results. + The evaluation results. """ # Get targets (gt) and predictions (dt) for the given image and category - gt = self._targets[img_id, cat_id] - dt = self._predictions[img_id, cat_id] + gt: list[dict[str, Any]] = self._targets[img_id, cat_id] + dt: list[dict[str, Any]] = self._predictions[img_id, cat_id] # If there is nothing to evaluate if len(gt) == 0 and len(dt) == 0: @@ -705,7 +744,7 @@ class COCOEvaluator: for g_idx, g in enumerate(gt): # If current gt is already matched, and not a crowd, continue # if gt_matches[tresh_idx, g_idx] > 0 and not iscrowd[g_idx]: - iscrowd = int(g.get("iscrowd")) + iscrowd = int(g.get("iscrowd", 0)) if gt_matches[tresh_idx, g_idx] > 0 and not iscrowd: continue # Stop searching the ground truths @@ -755,10 +794,7 @@ class COCOEvaluator: "dtIgnore": dt_ignore, } - def __str__(self): - self.summarize() - - def _accumulate(self): + def _accumulate(self) -> None: """ Accumulate per image evaluation results and store the result in self.results """ @@ -796,7 +832,9 @@ class COCOEvaluator: # Create sets for indexing set_categories = set(self.params.cat_ids) - set_area_ranges = set(map(tuple, self.params.area_range)) + set_area_ranges: set[tuple[float, ...]] = { + tuple(a) for a in self.params.area_range + } set_max_detections = set(self.params.max_dets) set_image_ids = set(self.params.img_ids) @@ -885,9 +923,9 @@ class COCOEvaluator: # Precision: TP / (FP + TP) pr = (tp / (fp + tp + EPS)).tolist() # List to compute the precision at each recall threshold - precision_at_recall = [0] * num_recall_thresholds + precision_at_recall = [0.0] * num_recall_thresholds # List to compute the score at each recall threshold - score_at_recall = [0] * num_recall_thresholds + score_at_recall = [0.0] * num_recall_thresholds # Set recall to either the final recall value or 0 (when there # is no TP) @@ -934,7 +972,9 @@ class COCOEvaluator: } # Helper function to compute average precision while handling -1 sentinel values - def compute_average_precision(precision_slice): + def compute_average_precision( + precision_slice: np.ndarray, + ) -> tuple[np.ndarray, np.ndarray]: """Compute average precision while handling -1 sentinel values.""" masked = np.ma.masked_equal(precision_slice, -1) if masked.count() == 0: @@ -1010,14 +1050,17 @@ class COCOEvaluator: "ap_per_class_large": ap_per_class_large, } - def _pycocotools_summarize(self): + def _pycocotools_summarize(self) -> None: """ Compute and display summary metrics for evaluation results. """ def _summarize( - use_ap: bool = True, iou_thr=None, area_range=ObjectSize.ALL, max_dets=100 - ): + use_ap: bool = True, + iou_thr: float | None = None, + area_range: ObjectSize = ObjectSize.ALL, + max_dets: int = 100, + ) -> float: iStr = " {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.10f}" titleStr = "Average Precision" if use_ap else "Average Recall" typeStr = "(AP)" if use_ap else "(AR)" @@ -1047,14 +1090,14 @@ class COCOEvaluator: s = s[t] s = s[:, :, area_range_idx, max_detections_idx] if len(s[s > -1]) == 0: - mean_s = -1 + mean_s = -1.0 else: - mean_s = np.mean(s[s > -1]) + mean_s = float(np.mean(s[s > -1])) print(iStr.format(titleStr, typeStr, iou_str, area_range, max_dets, mean_s)) return mean_s - def _summarize_predictions(): - stats = np.zeros((12,)) + def _summarize_predictions() -> np.ndarray: + stats: np.ndarray = np.zeros((12,)) stats[0] = _summarize(use_ap=True) stats[1] = _summarize( use_ap=True, iou_thr=0.5, max_dets=self.params.max_dets[2] @@ -1098,9 +1141,9 @@ class COCOEvaluator: return stats if len(self.results) != 0: - self.stats = _summarize_predictions() + self.stats = _summarize_predictions().tolist() - def evaluate(self): + def evaluate(self) -> None: """ Start the per image evaluation on all images and keeep results in self.eval_imgs (a list of dictionaries). @@ -1180,11 +1223,10 @@ class MeanAveragePrecision(Metric): Initialize the Mean Average Precision metric. Args: - metric_target (MetricTarget): The type of detection data to use. - class_agnostic (bool): Whether to treat all data as a single class. - class_mapping (Optional[Dict[int, int]]): A dictionary to map class IDs to - new IDs. - image_indices (Optional[List[int]]): The indices of the images to use. + metric_target: The type of detection data to use. + class_agnostic: Whether to treat all data as a single class. + class_mapping: A dictionary to map class IDs to new IDs. + image_indices: The indices of the images to use. """ self._metric_target = metric_target self._class_agnostic = class_agnostic @@ -1210,11 +1252,11 @@ class MeanAveragePrecision(Metric): Add new predictions and targets to the metric, but do not compute the result. Args: - predictions (Union[Detections, List[Detections]]): The predicted detections. - targets (Union[Detections, List[Detections]]): The ground-truth detections. + predictions: The predicted detections. + targets: The ground-truth detections. Returns: - (MeanAveragePrecision): The updated metric instance. + The updated metric instance. """ if not isinstance(predictions, list): predictions = [predictions] @@ -1232,51 +1274,67 @@ class MeanAveragePrecision(Metric): targets = deepcopy(targets) for prediction in predictions: - prediction.class_id[:] = -1 + if prediction.class_id is not None: + prediction.class_id[:] = -1 for target in targets: - target.class_id[:] = -1 + if target.class_id is not None: + target.class_id[:] = -1 self._predictions_list.extend(predictions) self._targets_list.extend(targets) return self - def _prepare_targets(self, targets): + def _prepare_targets( + self, targets: list[Detections] + ) -> dict[str, list[dict[str, Any]]]: """Transform targets into a dictionary that can be used by the COCO evaluator""" images = [{"id": img_id} for img_id in range(len(targets))] if self._image_indices is not None: - images = [ - {"id": self._image_indices[img_id.get("id")]} for img_id in images - ] + images = [{"id": self._image_indices[img["id"]]} for img in images] # Annotations list - annotations = [] + annotations: list[dict[str, Any]] = [] for image_id, image_targets in enumerate(targets): if self._image_indices is not None: image_id = self._image_indices[image_id] - for target_idx, target in enumerate(image_targets): - xyxy = target[0] # or xyxy = prediction[0]; xyxy[2:4] -= xyxy[0:2] + + # Ensure xyxy is not None + if image_targets.xyxy is None: + continue + + for target_idx, xyxy in enumerate(image_targets.xyxy): xywh = [xyxy[0], xyxy[1], xyxy[2] - xyxy[0], xyxy[3] - xyxy[1]] - # Get "area" and "iscrowd" (default 0) from data - data = target[5] - if self._class_mapping is not None: - category_id = self._class_mapping[target[3].item()] - else: - category_id = target[3].item() + # Default values + category_id = 0 + + if image_targets.class_id is not None: + cls_id = image_targets.class_id[target_idx] + if self._class_mapping is not None: + category_id = self._class_mapping[int(cls_id)] + else: + category_id = int(cls_id) + + # Use area from data if available, otherwise calculate from bbox + area = None + if image_targets.data is not None and "area" in image_targets.data: + area = float(image_targets.data["area"][target_idx]) - # Use area from data if available (e.g., COCO datasets) - # Otherwise use Detections.area property - area = data.get("area") if data else None if area is None: - area = image_targets.area[target_idx] + area = xywh[2] * xywh[3] + + iscrowd = 0 + if image_targets.data is not None and "iscrowd" in image_targets.data: + iscrowd = int(image_targets.data["iscrowd"][target_idx]) dict_annotation = { "area": area, - "iscrowd": data.get("iscrowd", 0), + "iscrowd": iscrowd, "image_id": image_id, "bbox": xywh, "category_id": category_id, "id": len(annotations) + 1, # Start IDs from 1 (0 means no match) + "ignore": 0, } annotations.append(dict_annotation) # Category list @@ -1289,25 +1347,53 @@ class MeanAveragePrecision(Metric): "categories": categories, } - def _prepare_predictions(self, predictions): + def _prepare_predictions( + self, predictions: list[Detections] + ) -> list[dict[str, Any]]: """Transform predictions into a list of predictions that can be used by the COCO evaluator.""" - coco_predictions = [] + coco_predictions: list[dict[str, Any]] = [] for image_id, image_predictions in enumerate(predictions): if self._image_indices is not None: image_id = self._image_indices[image_id] - for prediction in image_predictions: - xyxy = prediction[0] # or xyxy = prediction[0]; xyxy[2:4] -= xyxy[0:2] + + if image_predictions.xyxy is None: + continue + + for pred_idx, xyxy in enumerate(image_predictions.xyxy): xywh = [xyxy[0], xyxy[1], xyxy[2] - xyxy[0], xyxy[3] - xyxy[1]] - if self._class_mapping is not None: - category_id = self._class_mapping[prediction[3].item()] - else: - category_id = prediction[3].item() + + category_id = 0 + score = 0.0 + + if image_predictions.class_id is not None: + cls_id = image_predictions.class_id[pred_idx] + if self._class_mapping is not None: + category_id = self._class_mapping[int(cls_id)] + else: + category_id = int(cls_id) + + if image_predictions.confidence is not None: + score = float(image_predictions.confidence[pred_idx]) + + # Use area from data if available, otherwise calculate from bbox + area = None + if ( + image_predictions.data is not None + and "area" in image_predictions.data + ): + area = float(image_predictions.data["area"][pred_idx]) + + if area is None: + area = xywh[2] * xywh[3] + dict_prediction = { "image_id": image_id, "bbox": xywh, - "score": prediction[2].item(), + "score": score, "category_id": category_id, + "area": area, + "id": len(coco_predictions) + 1, } coco_predictions.append(dict_prediction) return coco_predictions @@ -1319,7 +1405,7 @@ class MeanAveragePrecision(Metric): Source: https://github.com/rafaelpadilla/review_object_detection_metrics Returns: - (MeanAveragePrecisionResult): The Mean Average Precision result. + The Mean Average Precision result. """ total_images_predictions = len(self._predictions_list) total_images_targets = len(self._targets_list) @@ -1349,7 +1435,7 @@ class MeanAveragePrecision(Metric): mAP_scores=cocoEval.results["mAP_scores_small"], ap_per_class=cocoEval.results["ap_per_class_small"], iou_thresholds=cocoEval.params.iou_thrs, - matched_classes=cocoEval.params.cat_ids, + matched_classes=np.array(cocoEval.params.cat_ids), ) # Create MeanAveragePrecisionResult object for medium objects mAP_medium = MeanAveragePrecisionResult( @@ -1358,7 +1444,7 @@ class MeanAveragePrecision(Metric): mAP_scores=cocoEval.results["mAP_scores_medium"], ap_per_class=cocoEval.results["ap_per_class_medium"], iou_thresholds=cocoEval.params.iou_thrs, - matched_classes=cocoEval.params.cat_ids, + matched_classes=np.array(cocoEval.params.cat_ids), ) # Create MeanAveragePrecisionResult object for large objects mAP_large = MeanAveragePrecisionResult( @@ -1367,7 +1453,7 @@ class MeanAveragePrecision(Metric): mAP_scores=cocoEval.results["mAP_scores_large"], ap_per_class=cocoEval.results["ap_per_class_large"], iou_thresholds=cocoEval.params.iou_thrs, - matched_classes=cocoEval.params.cat_ids, + matched_classes=np.array(cocoEval.params.cat_ids), ) # Create the final MeanAveragePrecisionResult object @@ -1377,7 +1463,7 @@ class MeanAveragePrecision(Metric): mAP_scores=cocoEval.results["mAP_scores_all_sizes"], ap_per_class=cocoEval.results["ap_per_class_all_sizes"], iou_thresholds=cocoEval.params.iou_thrs, - matched_classes=cocoEval.params.cat_ids, + matched_classes=np.array(cocoEval.params.cat_ids), small_objects=mAP_small, medium_objects=mAP_medium, large_objects=mAP_large, diff --git a/supervision/metrics/mean_average_recall.py b/supervision/metrics/mean_average_recall.py index 9ba3fe05..764d0f34 100644 --- a/supervision/metrics/mean_average_recall.py +++ b/supervision/metrics/mean_average_recall.py @@ -2,7 +2,7 @@ from __future__ import annotations from copy import deepcopy from dataclasses import dataclass -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import numpy as np from matplotlib import pyplot as plt @@ -26,6 +26,218 @@ if TYPE_CHECKING: import pandas as pd +@dataclass +class MeanAverageRecallResult: + """ + The results of the Mean Average Recall metric calculation. + + Defaults to `0` if no detections or targets were provided. + + Attributes: + metric_target: the type of data used for the metric - + boxes, masks or oriented bounding boxes. + mAR_at_1: the Mean Average Recall, when considering only the top + highest confidence detection for each class. + mAR_at_10: the Mean Average Recall, when considering top 10 + highest confidence detections for each class. + mAR_at_100: the Mean Average Recall, when considering top 100 + highest confidence detections for each class. + recall_per_class: the recall scores per class and IoU threshold. + Shape: `(num_target_classes, num_iou_thresholds)` + max_detections: the array with maximum number of detections + considered. + iou_thresholds: the IoU thresholds used in the calculations. + matched_classes: the class IDs of all matched classes. + Corresponds to the rows of `recall_per_class`. + small_objects: the Mean Average Recall + metric results for small objects (area < 32²). + medium_objects: the Mean Average Recall + metric results for medium objects (32² ≤ area < 96²). + large_objects: the Mean Average Recall + metric results for large objects (area ≥ 96²). + """ + + metric_target: MetricTarget + + @property + def mAR_at_1(self) -> float: + return float(self.recall_scores[0]) + + @property + def mAR_at_10(self) -> float: + return float(self.recall_scores[1]) + + @property + def mAR_at_100(self) -> float: + return float(self.recall_scores[2]) + + recall_scores: np.ndarray + recall_per_class: np.ndarray + max_detections: np.ndarray + iou_thresholds: np.ndarray + matched_classes: np.ndarray + + small_objects: MeanAverageRecallResult | None + medium_objects: MeanAverageRecallResult | None + large_objects: MeanAverageRecallResult | None + + def __str__(self) -> str: + """ + Format as a pretty string. + + Example: + ```python + print(mar_results) + # MeanAverageRecallResult: + # Metric target: MetricTarget.BOXES + # mAR @ 1: 0.1362 + # mAR @ 10: 0.4239 + # mAR @ 100: 0.5241 + # max detections: [1 10 100] + # IoU thresh: [0.5 0.55 0.6 ...] + # mAR per class: + # 0: [0.78571 0.78571 0.78571 ...] + # ... + # Small objects: ... + # Medium objects: ... + # Large objects: ... + ``` + """ + out_str = ( + f"{self.__class__.__name__}:\n" + f"Metric target: {self.metric_target}\n" + f"mAR @ 1: {self.mAR_at_1:.4f}\n" + f"mAR @ 10: {self.mAR_at_10:.4f}\n" + f"mAR @ 100: {self.mAR_at_100:.4f}\n" + f"max detections: {self.max_detections}\n" + f"IoU thresh: {self.iou_thresholds}\n" + f"mAR per class:\n" + ) + if self.recall_per_class.size == 0: + out_str += " No results\n" + for class_id, recall_of_class in zip( + self.matched_classes, self.recall_per_class + ): + out_str += f" {class_id}: {recall_of_class}\n" + + indent = " " + if self.small_objects is not None: + indented = indent + str(self.small_objects).replace("\n", f"\n{indent}") + out_str += f"\nSmall objects:\n{indented}" + if self.medium_objects is not None: + indented = indent + str(self.medium_objects).replace("\n", f"\n{indent}") + out_str += f"\nMedium objects:\n{indented}" + if self.large_objects is not None: + indented = indent + str(self.large_objects).replace("\n", f"\n{indent}") + out_str += f"\nLarge objects:\n{indented}" + + return out_str + + def to_pandas(self) -> pd.DataFrame: + """ + Convert the result to a pandas DataFrame. + + Returns: + The result as a DataFrame. + """ + ensure_pandas_installed() + import pandas as pd + + pandas_data = { + "mAR @ 1": self.mAR_at_1, + "mAR @ 10": self.mAR_at_10, + "mAR @ 100": self.mAR_at_100, + } + + if self.small_objects is not None: + small_objects_df = self.small_objects.to_pandas() + for key, value in small_objects_df.items(): + pandas_data[f"small_objects_{key}"] = value + if self.medium_objects is not None: + medium_objects_df = self.medium_objects.to_pandas() + for key, value in medium_objects_df.items(): + pandas_data[f"medium_objects_{key}"] = value + if self.large_objects is not None: + large_objects_df = self.large_objects.to_pandas() + for key, value in large_objects_df.items(): + pandas_data[f"large_objects_{key}"] = value + + return pd.DataFrame(pandas_data, index=[0]) + + def plot(self) -> None: + """ + Plot the Mean Average Recall results. + + ![example_plot](\ + https://media.roboflow.com/supervision-docs/metrics/mAR_plot_example.png\ + ){ align=center width="800" } + """ + labels = ["mAR @ 1", "mAR @ 10", "mAR @ 100"] + values = [self.mAR_at_1, self.mAR_at_10, self.mAR_at_100] + colors = [LEGACY_COLOR_PALETTE[0]] * 3 + + if self.small_objects is not None: + small_objects = self.small_objects + labels += ["Small: mAR @ 1", "Small: mAR @ 10", "Small: mAR @ 100"] + values += [ + small_objects.mAR_at_1, + small_objects.mAR_at_10, + small_objects.mAR_at_100, + ] + colors += [LEGACY_COLOR_PALETTE[3]] * 3 + + if self.medium_objects is not None: + medium_objects = self.medium_objects + labels += ["Medium: mAR @ 1", "Medium: mAR @ 10", "Medium: mAR @ 100"] + values += [ + medium_objects.mAR_at_1, + medium_objects.mAR_at_10, + medium_objects.mAR_at_100, + ] + colors += [LEGACY_COLOR_PALETTE[2]] * 3 + + if self.large_objects is not None: + large_objects = self.large_objects + labels += ["Large: mAR @ 1", "Large: mAR @ 10", "Large: mAR @ 100"] + values += [ + large_objects.mAR_at_1, + large_objects.mAR_at_10, + large_objects.mAR_at_100, + ] + colors += [LEGACY_COLOR_PALETTE[4]] * 3 + + plt.rcParams["font.family"] = "monospace" + + _, ax = plt.subplots(figsize=(10, 6)) + ax.set_ylim(0, 1) + ax.set_ylabel("Value", fontweight="bold") + title = ( + f"Mean Average Recall, by Object Size\n(target: {self.metric_target.value})" + ) + ax.set_title(title, fontweight="bold") + + x_positions = range(len(labels)) + bars = ax.bar(x_positions, values, color=colors, align="center") + + ax.set_xticks(x_positions) + ax.set_xticklabels(labels, rotation=45, ha="right") + + for bar in bars: + y_value = bar.get_height() + ax.text( + bar.get_x() + bar.get_width() / 2, + y_value + 0.02, + f"{y_value:.2f}", + ha="center", + va="bottom", + ) + + plt.rcParams["font.family"] = "sans-serif" + + plt.tight_layout() + plt.show() + + class MeanAverageRecall(Metric): """ Mean Average Recall (mAR) measures how well the model detects @@ -69,7 +281,7 @@ class MeanAverageRecall(Metric): Initialize the Mean Average Recall metric. Args: - metric_target (MetricTarget): The type of detection data to use. + metric_target: The type of detection data to use. """ self._metric_target = metric_target @@ -94,11 +306,11 @@ class MeanAverageRecall(Metric): Add new predictions and targets to the metric, but do not compute the result. Args: - predictions (Union[Detections, List[Detections]]): The predicted detections. - targets (Union[Detections, List[Detections]]): The target detections. + predictions: The predicted detections. + targets: The target detections. Returns: - (Recall): The updated metric instance. + The updated metric instance. """ if not isinstance(predictions, list): predictions = [predictions] @@ -122,7 +334,7 @@ class MeanAverageRecall(Metric): and ground-truth, at different IoU thresholds and maximum detection counts. Returns: - (MeanAverageRecallResult): The Mean Average Recall metric result. + The Mean Average Recall metric result. """ result = self._compute(self._predictions_list, self._targets_list) @@ -149,7 +361,7 @@ class MeanAverageRecall(Metric): self, predictions_list: list[Detections], targets_list: list[Detections] ) -> MeanAverageRecallResult: iou_thresholds = np.linspace(0.5, 0.95, 10) - stats = [] + stats: list[Any] = [] for predictions, targets in zip(predictions_list, targets_list): prediction_contents = self._detections_content(predictions) @@ -181,7 +393,14 @@ class MeanAverageRecall(Metric): ) matches = self._match_detection_batch( - predictions.class_id, targets.class_id, iou, iou_thresholds + predictions.class_id + if predictions.class_id is not None + else np.array([]), + targets.class_id + if targets.class_id is not None + else np.array([]), + iou, + iou_thresholds, ) stats.append( ( @@ -286,8 +505,8 @@ class MeanAverageRecall(Metric): matches = matches[np.unique(matches[:, 0], return_index=True)[1]] correct[matches[:, 1].astype(int), i] = True - - return correct + result: np.ndarray = correct + return result @staticmethod def _compute_confusion_matrix( @@ -304,20 +523,20 @@ class MeanAverageRecall(Metric): in descending order. Args: - sorted_matches: np.ndarray, bool, shape (P, Th), that is True + sorted_matches: shape (P, Th), that is True if the prediction is a true positive at the given IoU threshold. - sorted_prediction_class_ids: np.ndarray, int, shape (P,), containing + sorted_prediction_class_ids: shape (P,), containing the class id for each prediction. - unique_classes: np.ndarray, int, shape (C,), containing the unique + unique_classes: shape (C,), containing the unique class ids. - class_counts: np.ndarray, int, shape (C,), containing the number + class_counts: shape (C,), containing the number of true instances for each class. - max_detections: Optional[int], the maximum number of detections to + max_detections: The maximum number of detections to consider for each class. Extra detections are considered false positives. By default, all detections are considered. Returns: - np.ndarray, shape (C, Th, 3), containing the true positives, false + shape (C, Th, 3), containing the true positives, false positives, and false negatives for each class and IoU threshold. """ num_thresholds = sorted_matches.shape[1] @@ -343,12 +562,13 @@ class MeanAverageRecall(Metric): false_positives = (1 - limited_matches).sum(0) false_negatives = num_true - true_positives - false_negatives = num_true - true_positives + confusion_matrix[class_idx] = np.stack( [true_positives, false_positives, false_negatives], axis=1 ) - return confusion_matrix + result_confusion_matrix: np.ndarray = confusion_matrix + return result_confusion_matrix @staticmethod def _compute_recall(confusion_matrix: np.ndarray) -> np.ndarray: @@ -356,11 +576,11 @@ class MeanAverageRecall(Metric): Broadcastable function, computing the recall from the confusion matrix. Arguments: - confusion_matrix: np.ndarray, shape (N, ..., 3), where the last dimension + confusion_matrix: shape (N, ..., 3), where the last dimension contains the true positives, false positives, and false negatives. Returns: - np.ndarray, shape (N, ...), containing the recall for each element. + shape (N, ...), containing the recall for each element. """ if not confusion_matrix.shape[-1] == 3: raise ValueError( @@ -373,7 +593,8 @@ class MeanAverageRecall(Metric): denominator = true_positives + false_negatives recall = np.where(denominator == 0, 0, true_positives / denominator) - return recall + result_recall: np.ndarray = recall + return result_recall def _detections_content(self, detections: Detections) -> np.ndarray: """Return boxes, masks or oriented bounding boxes from detections.""" @@ -388,17 +609,24 @@ class MeanAverageRecall(Metric): if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: obb = detections.data.get(ORIENTED_BOX_COORDINATES) if obb is not None and len(obb) > 0: - return np.array(obb, dtype=np.float32) + result_obb: np.ndarray = np.array(obb, dtype=np.float32) + return result_obb return self._make_empty_content() raise ValueError(f"Invalid metric target: {self._metric_target}") def _make_empty_content(self) -> np.ndarray: if self._metric_target == MetricTarget.BOXES: - return np.empty((0, 4), dtype=np.float32) + empty_boxes: np.ndarray = np.empty((0, 4), dtype=np.float32) + return empty_boxes + if self._metric_target == MetricTarget.MASKS: - return np.empty((0, 0, 0), dtype=bool) + empty_masks: np.ndarray = np.empty((0, 0, 0), dtype=bool) + return empty_masks + if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: - return np.empty((0, 4, 2), dtype=np.float32) + empty_obb: np.ndarray = np.empty((0, 4, 2), dtype=np.float32) + return empty_obb + raise ValueError(f"Invalid metric target: {self._metric_target}") def _filter_detections_by_size( @@ -433,6 +661,9 @@ class MeanAverageRecall(Metric): targets_list: list[Detections], size_category: ObjectSizeCategory, ) -> tuple[list[Detections], list[Detections]]: + """ + Filter predictions and targets by object size category. + """ new_predictions_list = [] new_targets_list = [] for predictions, targets in zip(predictions_list, targets_list): @@ -443,240 +674,3 @@ class MeanAverageRecall(Metric): self._filter_detections_by_size(targets, size_category) ) return new_predictions_list, new_targets_list - - -@dataclass -class MeanAverageRecallResult: - # """ - # The results of the recall metric calculation. - - # Defaults to `0` if no detections or targets were provided. - - # Attributes: - # metric_target (MetricTarget): the type of data used for the metric - - # boxes, masks or oriented bounding boxes. - # averaging_method (AveragingMethod): the averaging method used to compute the - # recall. Determines how the recall is aggregated across classes. - # recall_at_50 (float): the recall at IoU threshold of `0.5`. - # recall_at_75 (float): the recall at IoU threshold of `0.75`. - # recall_scores (np.ndarray): the recall scores at each IoU threshold. - # Shape: `(num_iou_thresholds,)` - # recall_per_class (np.ndarray): the recall scores per class and IoU threshold. - # Shape: `(num_target_classes, num_iou_thresholds)` - # iou_thresholds (np.ndarray): the IoU thresholds used in the calculations. - # matched_classes (np.ndarray): the class IDs of all matched classes. - # Corresponds to the rows of `recall_per_class`. - # small_objects (Optional[RecallResult]): the Recall metric results - # for small objects. - # medium_objects (Optional[RecallResult]): the Recall metric results - # for medium objects. - # large_objects (Optional[RecallResult]): the Recall metric results - # for large objects. - # """ - """ - The results of the Mean Average Recall metric calculation. - - Defaults to `0` if no detections or targets were provided. - - Attributes: - metric_target (MetricTarget): the type of data used for the metric - - boxes, masks or oriented bounding boxes. - mAR_at_1 (float): the Mean Average Recall, when considering only the top - highest confidence detection for each class. - mAR_at_10 (float): the Mean Average Recall, when considering top 10 - highest confidence detections for each class. - mAR_at_100 (float): the Mean Average Recall, when considering top 100 - highest confidence detections for each class. - recall_per_class (np.ndarray): the recall scores per class and IoU threshold. - Shape: `(num_target_classes, num_iou_thresholds)` - max_detections (np.ndarray): the array with maximum number of detections - considered. - iou_thresholds (np.ndarray): the IoU thresholds used in the calculations. - matched_classes (np.ndarray): the class IDs of all matched classes. - Corresponds to the rows of `recall_per_class`. - small_objects (Optional[MeanAverageRecallResult]): the Mean Average Recall - metric results for small objects (area < 32²). - medium_objects (Optional[MeanAverageRecallResult]): the Mean Average Recall - metric results for medium objects (32² ≤ area < 96²). - large_objects (Optional[MeanAverageRecallResult]): the Mean Average Recall - metric results for large objects (area ≥ 96²). - """ - - metric_target: MetricTarget - - @property - def mAR_at_1(self) -> float: - return self.recall_scores[0] - - @property - def mAR_at_10(self) -> float: - return self.recall_scores[1] - - @property - def mAR_at_100(self) -> float: - return self.recall_scores[2] - - recall_scores: np.ndarray - recall_per_class: np.ndarray - max_detections: np.ndarray - iou_thresholds: np.ndarray - matched_classes: np.ndarray - - small_objects: MeanAverageRecallResult | None - medium_objects: MeanAverageRecallResult | None - large_objects: MeanAverageRecallResult | None - - def __str__(self) -> str: - """ - Format as a pretty string. - - Example: - ```python - # MeanAverageRecallResult: - # Metric target: MetricTarget.BOXES - # mAR @ 1: 0.1362 - # mAR @ 10: 0.4239 - # mAR @ 100: 0.5241 - # max detections: [1 10 100] - # IoU thresh: [0.5 0.55 0.6 ...] - # mAR per class: - # 0: [0.78571 0.78571 0.78571 ...] - # ... - # Small objects: ... - # Medium objects: ... - # Large objects: ... - ``` - """ - out_str = ( - f"{self.__class__.__name__}:\n" - f"Metric target: {self.metric_target}\n" - f"mAR @ 1: {self.mAR_at_1:.4f}\n" - f"mAR @ 10: {self.mAR_at_10:.4f}\n" - f"mAR @ 100: {self.mAR_at_100:.4f}\n" - f"max detections: {self.max_detections}\n" - f"IoU thresh: {self.iou_thresholds}\n" - f"mAR per class:\n" - ) - if self.recall_per_class.size == 0: - out_str += " No results\n" - for class_id, recall_of_class in zip( - self.matched_classes, self.recall_per_class - ): - out_str += f" {class_id}: {recall_of_class}\n" - - indent = " " - if self.small_objects is not None: - indented = indent + str(self.small_objects).replace("\n", f"\n{indent}") - out_str += f"\nSmall objects:\n{indented}" - if self.medium_objects is not None: - indented = indent + str(self.medium_objects).replace("\n", f"\n{indent}") - out_str += f"\nMedium objects:\n{indented}" - if self.large_objects is not None: - indented = indent + str(self.large_objects).replace("\n", f"\n{indent}") - out_str += f"\nLarge objects:\n{indented}" - - return out_str - - def to_pandas(self) -> pd.DataFrame: - """ - Convert the result to a pandas DataFrame. - - Returns: - (pd.DataFrame): The result as a DataFrame. - """ - ensure_pandas_installed() - import pandas as pd - - pandas_data = { - "mAR @ 1": self.mAR_at_1, - "mAR @ 10": self.mAR_at_10, - "mAR @ 100": self.mAR_at_100, - } - - if self.small_objects is not None: - small_objects_df = self.small_objects.to_pandas() - for key, value in small_objects_df.items(): - pandas_data[f"small_objects_{key}"] = value - if self.medium_objects is not None: - medium_objects_df = self.medium_objects.to_pandas() - for key, value in medium_objects_df.items(): - pandas_data[f"medium_objects_{key}"] = value - if self.large_objects is not None: - large_objects_df = self.large_objects.to_pandas() - for key, value in large_objects_df.items(): - pandas_data[f"large_objects_{key}"] = value - - return pd.DataFrame(pandas_data, index=[0]) - - def plot(self): - """ - Plot the Mean Average Recall results. - - ![example_plot]( - https://media.roboflow.com/supervision-docs/metrics/mAR_plot_example.png - ){ align=center width="800" } - """ - labels = ["mAR @ 1", "mAR @ 10", "mAR @ 100"] - values = [self.mAR_at_1, self.mAR_at_10, self.mAR_at_100] - colors = [LEGACY_COLOR_PALETTE[0]] * 3 - - if self.small_objects is not None: - small_objects = self.small_objects - labels += ["Small: mAR @ 1", "Small: mAR @ 10", "Small: mAR @ 100"] - values += [ - small_objects.mAR_at_1, - small_objects.mAR_at_10, - small_objects.mAR_at_100, - ] - colors += [LEGACY_COLOR_PALETTE[3]] * 3 - - if self.medium_objects is not None: - medium_objects = self.medium_objects - labels += ["Medium: mAR @ 1", "Medium: mAR @ 10", "Medium: mAR @ 100"] - values += [ - medium_objects.mAR_at_1, - medium_objects.mAR_at_10, - medium_objects.mAR_at_100, - ] - colors += [LEGACY_COLOR_PALETTE[2]] * 3 - - if self.large_objects is not None: - large_objects = self.large_objects - labels += ["Large: mAR @ 1", "Large: mAR @ 10", "Large: mAR @ 100"] - values += [ - large_objects.mAR_at_1, - large_objects.mAR_at_10, - large_objects.mAR_at_100, - ] - colors += [LEGACY_COLOR_PALETTE[4]] * 3 - - plt.rcParams["font.family"] = "monospace" - - _, ax = plt.subplots(figsize=(10, 6)) - ax.set_ylim(0, 1) - ax.set_ylabel("Value", fontweight="bold") - title = ( - f"Mean Average Recall, by Object Size\n(target: {self.metric_target.value})" - ) - ax.set_title(title, fontweight="bold") - - x_positions = range(len(labels)) - bars = ax.bar(x_positions, values, color=colors, align="center") - - ax.set_xticks(x_positions) - ax.set_xticklabels(labels, rotation=45, ha="right") - - for bar in bars: - y_value = bar.get_height() - ax.text( - bar.get_x() + bar.get_width() / 2, - y_value + 0.02, - f"{y_value:.2f}", - ha="center", - va="bottom", - ) - - plt.rcParams["font.family"] = "sans-serif" - - plt.tight_layout() - plt.show() diff --git a/supervision/metrics/precision.py b/supervision/metrics/precision.py index 0f9f38db..618d698e 100644 --- a/supervision/metrics/precision.py +++ b/supervision/metrics/precision.py @@ -2,7 +2,7 @@ from __future__ import annotations from copy import deepcopy from dataclasses import dataclass -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import numpy as np from matplotlib import pyplot as plt @@ -71,8 +71,8 @@ class Precision(Metric): Initialize the Precision metric. Args: - metric_target (MetricTarget): The type of detection data to use. - averaging_method (AveragingMethod): The averaging method used to compute the + metric_target: The type of detection data to use. + averaging_method: The averaging method used to compute the precision. Determines how the precision is aggregated across classes. """ self._metric_target = metric_target @@ -97,11 +97,11 @@ class Precision(Metric): Add new predictions and targets to the metric, but do not compute the result. Args: - predictions (Union[Detections, List[Detections]]): The predicted detections. - targets (Union[Detections, List[Detections]]): The target detections. + predictions: The predicted detections. + targets: The target detections. Returns: - (Precision): The updated metric instance. + The updated metric instance. """ if not isinstance(predictions, list): predictions = [predictions] @@ -125,7 +125,7 @@ class Precision(Metric): data, at different IoU thresholds. Returns: - (PrecisionResult): The precision metric result. + The precision metric result. """ result = self._compute(self._predictions_list, self._targets_list) @@ -152,7 +152,7 @@ class Precision(Metric): self, predictions_list: list[Detections], targets_list: list[Detections] ) -> PrecisionResult: iou_thresholds = np.linspace(0.5, 0.95, 10) - stats = [] + stats: list[Any] = [] for predictions, targets in zip(predictions_list, targets_list): prediction_contents = self._detections_content(predictions) @@ -184,7 +184,14 @@ class Precision(Metric): ) matches = self._match_detection_batch( - predictions.class_id, targets.class_id, iou, iou_thresholds + predictions.class_id + if predictions.class_id is not None + else np.array([]), + targets.class_id + if targets.class_id is not None + else np.array([]), + iou, + iou_thresholds, ) stats.append( ( @@ -287,8 +294,8 @@ class Precision(Metric): matches = matches[np.unique(matches[:, 0], return_index=True)[1]] correct[matches[:, 1].astype(int), i] = True - - return correct + result: np.ndarray = correct + return result @staticmethod def _compute_confusion_matrix( @@ -303,18 +310,18 @@ class Precision(Metric): Assumes the matches and prediction_class_ids are sorted by confidence in descending order. - Arguments: - sorted_matches: np.ndarray, bool, shape (P, Th), that is True + Args: + sorted_matches: shape (P, Th), that is True if the prediction is a true positive at the given IoU threshold. - sorted_prediction_class_ids: np.ndarray, int, shape (P,), containing + sorted_prediction_class_ids: shape (P,), containing the class id for each prediction. - unique_classes: np.ndarray, int, shape (C,), containing the unique + unique_classes: shape (C,), containing the unique class ids. - class_counts: np.ndarray, int, shape (C,), containing the number + class_counts: shape (C,), containing the number of true instances for each class. Returns: - np.ndarray, shape (C, Th, 3), containing the true positives, false + shape (C, Th, 3), containing the true positives, false positives, and false negatives for each class and IoU threshold. """ @@ -342,8 +349,8 @@ class Precision(Metric): confusion_matrix[class_idx] = np.stack( [true_positives, false_positives, false_negatives], axis=1 ) - - return confusion_matrix + result_matrix: np.ndarray = confusion_matrix + return result_matrix @staticmethod def _compute_precision(confusion_matrix: np.ndarray) -> np.ndarray: @@ -351,11 +358,11 @@ class Precision(Metric): Broadcastable function, computing the precision from the confusion matrix. Arguments: - confusion_matrix: np.ndarray, shape (N, ..., 3), where the last dimension + confusion_matrix: shape (N, ..., 3), where the last dimension contains the true positives, false positives, and false negatives. Returns: - np.ndarray, shape (N, ...), containing the precision for each element. + shape (N, ...), containing the precision for each element. """ if not confusion_matrix.shape[-1] == 3: raise ValueError( @@ -368,7 +375,8 @@ class Precision(Metric): denominator = true_positives + false_positives precision = np.where(denominator == 0, 0, true_positives / denominator) - return precision + result_precision: np.ndarray = precision + return result_precision def _detections_content(self, detections: Detections) -> np.ndarray: """Return boxes, masks or oriented bounding boxes from detections.""" @@ -383,17 +391,24 @@ class Precision(Metric): if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: obb = detections.data.get(ORIENTED_BOX_COORDINATES) if obb is not None and len(obb) > 0: - return np.array(obb, dtype=np.float32) + result_obb: np.ndarray = np.array(obb, dtype=np.float32) + return result_obb return self._make_empty_content() raise ValueError(f"Invalid metric target: {self._metric_target}") def _make_empty_content(self) -> np.ndarray: if self._metric_target == MetricTarget.BOXES: - return np.empty((0, 4), dtype=np.float32) + empty_boxes: np.ndarray = np.empty((0, 4), dtype=np.float32) + return empty_boxes + if self._metric_target == MetricTarget.MASKS: - return np.empty((0, 0, 0), dtype=bool) + empty_masks: np.ndarray = np.empty((0, 0, 0), dtype=bool) + return empty_masks + if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: - return np.empty((0, 4, 2), dtype=np.float32) + empty_obb: np.ndarray = np.empty((0, 4, 2), dtype=np.float32) + return empty_obb + raise ValueError(f"Invalid metric target: {self._metric_target}") def _filter_detections_by_size( @@ -451,24 +466,24 @@ class PrecisionResult: Defaults to `0` if no detections or targets were provided. Attributes: - metric_target (MetricTarget): the type of data used for the metric - + metric_target: the type of data used for the metric - boxes, masks or oriented bounding boxes. - averaging_method (AveragingMethod): the averaging method used to compute the + averaging_method: the averaging method used to compute the precision. Determines how the precision is aggregated across classes. - precision_at_50 (float): the precision at IoU threshold of `0.5`. - precision_at_75 (float): the precision at IoU threshold of `0.75`. - precision_scores (np.ndarray): the precision scores at each IoU threshold. + precision_at_50: the precision at IoU threshold of `0.5`. + precision_at_75: the precision at IoU threshold of `0.75`. + precision_scores: the precision scores at each IoU threshold. Shape: `(num_iou_thresholds,)` - precision_per_class (np.ndarray): the precision scores per class and + precision_per_class: the precision scores per class and IoU threshold. Shape: `(num_target_classes, num_iou_thresholds)` - iou_thresholds (np.ndarray): the IoU thresholds used in the calculations. - matched_classes (np.ndarray): the class IDs of all matched classes. + iou_thresholds: the IoU thresholds used in the calculations. + matched_classes: the class IDs of all matched classes. Corresponds to the rows of `precision_per_class`. - small_objects (Optional[PrecisionResult]): the Precision metric results + small_objects: the Precision metric results for small objects (area < 32²). - medium_objects (Optional[PrecisionResult]): the Precision metric results + medium_objects: the Precision metric results for medium objects (32² ≤ area < 96²). - large_objects (Optional[PrecisionResult]): the Precision metric results + large_objects: the Precision metric results for large objects (area ≥ 96²). """ @@ -477,11 +492,11 @@ class PrecisionResult: @property def precision_at_50(self) -> float: - return self.precision_scores[0] + return float(self.precision_scores[0]) @property def precision_at_75(self) -> float: - return self.precision_scores[5] + return float(self.precision_scores[5]) precision_scores: np.ndarray precision_per_class: np.ndarray @@ -549,7 +564,7 @@ class PrecisionResult: Convert the result to a pandas DataFrame. Returns: - (pd.DataFrame): The result as a DataFrame. + The result as a DataFrame. """ ensure_pandas_installed() import pandas as pd @@ -574,7 +589,7 @@ class PrecisionResult: return pd.DataFrame(pandas_data, index=[0]) - def plot(self): + def plot(self) -> None: """ Plot the precision results. diff --git a/supervision/metrics/recall.py b/supervision/metrics/recall.py index 6986a6a0..66e619e7 100644 --- a/supervision/metrics/recall.py +++ b/supervision/metrics/recall.py @@ -2,7 +2,7 @@ from __future__ import annotations from copy import deepcopy from dataclasses import dataclass -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import numpy as np from matplotlib import pyplot as plt @@ -71,8 +71,8 @@ class Recall(Metric): Initialize the Recall metric. Args: - metric_target (MetricTarget): The type of detection data to use. - averaging_method (AveragingMethod): The averaging method used to compute the + metric_target: The type of detection data to use. + averaging_method: The averaging method used to compute the recall. Determines how the recall is aggregated across classes. """ self._metric_target = metric_target @@ -97,11 +97,11 @@ class Recall(Metric): Add new predictions and targets to the metric, but do not compute the result. Args: - predictions (Union[Detections, List[Detections]]): The predicted detections. - targets (Union[Detections, List[Detections]]): The target detections. + predictions: The predicted detections. + targets: The target detections. Returns: - (Recall): The updated metric instance. + The updated metric instance. """ if not isinstance(predictions, list): predictions = [predictions] @@ -125,7 +125,7 @@ class Recall(Metric): data, at different IoU thresholds. Returns: - (RecallResult): The recall metric result. + The recall metric result. """ result = self._compute(self._predictions_list, self._targets_list) @@ -152,7 +152,7 @@ class Recall(Metric): self, predictions_list: list[Detections], targets_list: list[Detections] ) -> RecallResult: iou_thresholds = np.linspace(0.5, 0.95, 10) - stats = [] + stats: list[Any] = [] for predictions, targets in zip(predictions_list, targets_list): prediction_contents = self._detections_content(predictions) @@ -184,7 +184,14 @@ class Recall(Metric): ) matches = self._match_detection_batch( - predictions.class_id, targets.class_id, iou, iou_thresholds + predictions.class_id + if predictions.class_id is not None + else np.array([]), + targets.class_id + if targets.class_id is not None + else np.array([]), + iou, + iou_thresholds, ) stats.append( ( @@ -285,8 +292,8 @@ class Recall(Metric): matches = matches[np.unique(matches[:, 0], return_index=True)[1]] correct[matches[:, 1].astype(int), i] = True - - return correct + result: np.ndarray = correct + return result @staticmethod def _compute_confusion_matrix( @@ -301,18 +308,18 @@ class Recall(Metric): Assumes the matches and prediction_class_ids are sorted by confidence in descending order. - Arguments: - sorted_matches: np.ndarray, bool, shape (P, Th), that is True + Args: + sorted_matches: shape (P, Th), that is True if the prediction is a true positive at the given IoU threshold. - sorted_prediction_class_ids: np.ndarray, int, shape (P,), containing + sorted_prediction_class_ids: shape (P,), containing the class id for each prediction. - unique_classes: np.ndarray, int, shape (C,), containing the unique + unique_classes: shape (C,), containing the unique class ids. - class_counts: np.ndarray, int, shape (C,), containing the number + class_counts: shape (C,), containing the number of true instances for each class. Returns: - np.ndarray, shape (C, Th, 3), containing the true positives, false + shape (C, Th, 3), containing the true positives, false positives, and false negatives for each class and IoU threshold. """ @@ -340,8 +347,8 @@ class Recall(Metric): confusion_matrix[class_idx] = np.stack( [true_positives, false_positives, false_negatives], axis=1 ) - - return confusion_matrix + result: np.ndarray = confusion_matrix + return result @staticmethod def _compute_recall(confusion_matrix: np.ndarray) -> np.ndarray: @@ -349,11 +356,11 @@ class Recall(Metric): Broadcastable function, computing the recall from the confusion matrix. Arguments: - confusion_matrix: np.ndarray, shape (N, ..., 3), where the last dimension + confusion_matrix: shape (N, ..., 3), where the last dimension contains the true positives, false positives, and false negatives. Returns: - np.ndarray, shape (N, ...), containing the recall for each element. + shape (N, ...), containing the recall for each element. """ if not confusion_matrix.shape[-1] == 3: raise ValueError( @@ -366,7 +373,8 @@ class Recall(Metric): denominator = true_positives + false_negatives recall = np.where(denominator == 0, 0, true_positives / denominator) - return recall + result: np.ndarray = recall + return result def _detections_content(self, detections: Detections) -> np.ndarray: """Return boxes, masks or oriented bounding boxes from detections.""" @@ -381,17 +389,24 @@ class Recall(Metric): if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: obb = detections.data.get(ORIENTED_BOX_COORDINATES) if obb is not None and len(obb) > 0: - return np.array(obb, dtype=np.float32) + result: np.ndarray = np.array(obb, dtype=np.float32) + return result return self._make_empty_content() raise ValueError(f"Invalid metric target: {self._metric_target}") def _make_empty_content(self) -> np.ndarray: if self._metric_target == MetricTarget.BOXES: - return np.empty((0, 4), dtype=np.float32) + empty_boxes: np.ndarray = np.empty((0, 4), dtype=np.float32) + return empty_boxes + if self._metric_target == MetricTarget.MASKS: - return np.empty((0, 0, 0), dtype=bool) + empty_masks: np.ndarray = np.empty((0, 0, 0), dtype=bool) + return empty_masks + if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: - return np.empty((0, 4, 2), dtype=np.float32) + empty_obb: np.ndarray = np.empty((0, 4, 2), dtype=np.float32) + return empty_obb + raise ValueError(f"Invalid metric target: {self._metric_target}") def _filter_detections_by_size( @@ -449,24 +464,24 @@ class RecallResult: Defaults to `0` if no detections or targets were provided. Attributes: - metric_target (MetricTarget): the type of data used for the metric - + metric_target: the type of data used for the metric - boxes, masks or oriented bounding boxes. - averaging_method (AveragingMethod): the averaging method used to compute the + averaging_method: the averaging method used to compute the recall. Determines how the recall is aggregated across classes. - recall_at_50 (float): the recall at IoU threshold of `0.5`. - recall_at_75 (float): the recall at IoU threshold of `0.75`. - recall_scores (np.ndarray): the recall scores at each IoU threshold. + recall_at_50: the recall at IoU threshold of `0.5`. + recall_at_75: the recall at IoU threshold of `0.75`. + recall_scores: the recall scores at each IoU threshold. Shape: `(num_iou_thresholds,)` - recall_per_class (np.ndarray): the recall scores per class and IoU threshold. + recall_per_class: the recall scores per class and IoU threshold. Shape: `(num_target_classes, num_iou_thresholds)` - iou_thresholds (np.ndarray): the IoU thresholds used in the calculations. - matched_classes (np.ndarray): the class IDs of all matched classes. + iou_thresholds: the IoU thresholds used in the calculations. + matched_classes: the class IDs of all matched classes. Corresponds to the rows of `recall_per_class`. - small_objects (Optional[RecallResult]): the Recall metric results + small_objects: the Recall metric results for small objects (area < 32²). - medium_objects (Optional[RecallResult]): the Recall metric results + medium_objects: the Recall metric results for medium objects (32² ≤ area < 96²). - large_objects (Optional[RecallResult]): the Recall metric results + large_objects: the Recall metric results for large objects (area ≥ 96²). """ @@ -475,11 +490,11 @@ class RecallResult: @property def recall_at_50(self) -> float: - return self.recall_scores[0] + return float(self.recall_scores[0]) @property def recall_at_75(self) -> float: - return self.recall_scores[5] + return float(self.recall_scores[5]) recall_scores: np.ndarray recall_per_class: np.ndarray @@ -547,7 +562,7 @@ class RecallResult: Convert the result to a pandas DataFrame. Returns: - (pd.DataFrame): The result as a DataFrame. + The result as a DataFrame. """ ensure_pandas_installed() import pandas as pd @@ -572,7 +587,7 @@ class RecallResult: return pd.DataFrame(pandas_data, index=[0]) - def plot(self): + def plot(self) -> None: """ Plot the recall results. diff --git a/supervision/metrics/utils/utils.py b/supervision/metrics/utils/utils.py index 9128fbe5..adb5ddf4 100644 --- a/supervision/metrics/utils/utils.py +++ b/supervision/metrics/utils/utils.py @@ -1,4 +1,4 @@ -def ensure_pandas_installed(): +def ensure_pandas_installed() -> None: try: import pandas # noqa except ImportError: